diff --git a/cachedb/cachedb.h b/cachedb/cachedb.h index 28922a6d042..5ed522bb120 100644 --- a/cachedb/cachedb.h +++ b/cachedb/cachedb.h @@ -162,7 +162,36 @@ typedef struct cachedb_funcs_t { */ int (*is_replicated) (cachedb_con *con); + int capability; + + /** + * OPTIONAL, advertised by CACHEDB_CAP_GET_BUF. Reads a value into a + * caller-owned buffer, so a hot read path need not allocate: it is + * get() without the pkg_malloc the caller would then have to free. + * + * @buf must be memory private to the calling process (its own stack or + * pkg). A backend may write into it speculatively and then abandon the + * attempt, so on ANY outcome other than a hit its contents are + * undefined - never a stale previous value to fall back on. The value + * is not NUL-terminated. + * + * Return values: + * -3: @buf is too small; *vlen is 0 and *needed holds the size that + * would be required (the caller may then fall back to get()) + * -2: key does not exist, or has expired + * -1: internal error, or a malformed request (NULL @buf, or @buflen + * below the backend's documented minimum) + * 0: found; *vlen bytes were written to @buf, always <= @buflen + * + * *vlen and *needed are zeroed before anything else is done, so a + * caller that ignores the return code reads a zero length rather than + * an uninitialised one. @needed may be NULL. NOTE this differs from + * get(), which signals a hit with a positive value at the script + * boundary - a hit here is 0. + */ + int (*get_buf) (cachedb_con *con, str *attr, char *buf, + unsigned int buflen, unsigned int *vlen, unsigned int *needed); } cachedb_funcs; typedef struct cachedb_engines { diff --git a/cachedb/cachedb_cap.h b/cachedb/cachedb_cap.h index 1b7b8c14930..eb017928c6c 100644 --- a/cachedb/cachedb_cap.h +++ b/cachedb/cachedb_cap.h @@ -51,8 +51,31 @@ typedef enum { CACHEDB_CAP_MAP_REMOVE = 1<<13, CACHEDB_CAP_MAP = (CACHEDB_CAP_MAP_GET|CACHEDB_CAP_MAP_SET|CACHEDB_CAP_MAP_REMOVE), + + /* backend implements get_buf() - an allocation-free read into a + * caller-owned buffer. Optional: every backend still provides get() */ + CACHEDB_CAP_GET_BUF = 1<<14, } cachedb_cap; +/* + * Preprocessor-visible companion to CACHEDB_CAP_GET_BUF. A consumer cannot test + * for the endpoint at runtime alone: get_buf is a struct member, so referencing it + * fails to compile against a core that predates it, and the capability above is an + * enum constant the preprocessor cannot see. This lets a module compile against + * either core and pick the allocation-free path up automatically: + * + * #ifdef CACHEDB_HAVE_GET_BUF + * if (cdbf.get_buf && CACHEDB_CAPABILITY(&cdbf, CACHEDB_CAP_GET_BUF)) + * ... use it ... + * else + * #endif + * ... use get() ... + * + * The runtime half stays necessary: a core may provide the endpoint while the + * configured backend does not implement it. + */ +#define CACHEDB_HAVE_GET_BUF 1 + #define CACHEDB_CAPABILITY(cdbf,cpv) (((cdbf)->capability & (cpv)) == (cpv)) static inline int check_cachedb_api(cachedb_engine *cde) diff --git a/modules/cachedb_perf/Makefile b/modules/cachedb_perf/Makefile new file mode 100644 index 00000000000..7f4eb632b2b --- /dev/null +++ b/modules/cachedb_perf/Makefile @@ -0,0 +1,18 @@ +# cachedb_perf module +# +# WARNING: do not run this directly, it should be run by the master Makefile + +include ../../Makefile.defs +auto_gen= +NAME=cachedb_perf.so + +# The clusterer_controller pull transport is compiled only when that module is +# part of this build - the top-level Makefile exports CLUSTERER_CTRL_SUPPORT=1 +# in that case (the clusterer module keys off the same hook). Without it, +# pulls and syncs use the clusterer module's bin links only, and an explicit +# pull_transport=clctr warns and degrades to bin. +ifeq ($(CLUSTERER_CTRL_SUPPORT),1) +DEFS+= -DCLUSTERER_CTRL_SUPPORT +endif + +include ../../Makefile.modules diff --git a/modules/cachedb_perf/README b/modules/cachedb_perf/README new file mode 100644 index 00000000000..9320a1f00b8 --- /dev/null +++ b/modules/cachedb_perf/README @@ -0,0 +1,1021 @@ +cachedb_perf Module + __________________________________________________________ + + Table of Contents + + 1. Admin Guide + + 1.1. Overview + 1.2. Dependencies + + 1.2.1. OpenSIPS Modules + 1.2.2. External Libraries or Applications + + 1.3. Exported Parameters + + 1.3.1. cache_collections (string) + 1.3.2. expiry_sweep_period (integer) + 1.3.3. cachedb_url (string) + 1.3.4. growth_load_factor (integer) + 1.3.5. growth_budget (integer) + 1.3.6. arena_hugepage_mb (integer) + 1.3.7. arena_selftest (integer) + 1.3.8. htable_selftest (integer) + 1.3.9. event_expired_collections (string) + 1.3.10. db_url (string) + 1.3.11. db_table (string) + 1.3.12. db_mode (integer) + 1.3.13. persist_collections (string) + 1.3.14. sync_cluster_id (integer) + 1.3.15. sync_shtag (string) + 1.3.16. replicate_collections (string) + 1.3.17. pull_transport (string) + 1.3.18. pull_timeout_ms (integer) + 1.3.19. pull_negative_ms (integer) + 1.3.20. pull_on_miss (integer) + + 1.4. DB persistence + 1.5. Cluster sync + 1.6. Degraded operation: what each surface does without + its modules + + 1.7. Exported Statistics + 1.8. Exported MI Functions + + 1.8.1. perf_stats + 1.8.2. perf_stats_reset + 1.8.3. Key introspection: perf_keys / perf_scan / + perf_dump / perf_get / perf_probe / + perf_pull / perf_set / perf_ttl / perf_del + + 1.8.4. perf_save / perf_load + 1.8.5. perf_sync + + 1.9. Exported Events + + 1.9.1. E_CACHEDB_PERF_EXPIRED + 1.9.2. E_CACHEDB_PERF_NOMEM + 1.9.3. E_CACHEDB_PERF_GROWN + 1.9.4. E_CACHEDB_PERF_MEM_DEGRADED + 1.9.5. E_CACHEDB_PERF_SYNCED + + 1.10. Exported Functions + + 1.10.1. perf_del(glob[, collection]) + 1.10.2. perf_mget(glob, keys_avp, vals_avp[, + collection[, limit]]) + + 1.10.3. perf_mget_json(glob, dst_var[, collection[, + limit]]) + + List of Examples + + 1.1. Set cache_collections parameter + 1.2. Set expiry_sweep_period parameter + 1.3. Set cachedb_url parameter + 1.4. Set growth_load_factor parameter + 1.5. Set growth_budget parameter + 1.6. Set arena_hugepage_mb parameter + 1.7. Set arena_selftest parameter + 1.8. Set htable_selftest parameter + 1.9. Set event_expired_collections parameter + 1.10. Set db_url parameter + 1.11. Set db_mode parameter + 1.12. Set persist_collections parameter + 1.13. Set sync_cluster_id parameter + 1.14. Setting sync_shtag + 1.15. Setting replicate_collections + 1.16. Cluster-sync setup (one authority, two read replicas) + 1.17. perf_stats usage + 1.18. perf_stats_reset usage + 1.19. introspection usage + 1.20. perf_save / perf_load usage + 1.21. perf_sync usage + 1.22. perf_del() usage + 1.23. perf_mget() usage + 1.24. perf_mget_json() usage + +Chapter 1. Admin Guide + +1.1. Overview + + This module is a high-performance local memory cache + implementing the Key-Value interface exported by the OpenSIPS + core. It is a drop-in alternative to cachedb_local, selected by + URL scheme (perf:// instead of local://), designed for large, + high-churn caches: lock-free reads, cache-line-sized buckets + and a table that grows at runtime instead of being sized once + at startup. + + Each OpenSIPS instance keeps its own in-memory copy, but a + whole collection can be persisted to an SQL backend so it + survives a restart (see DB persistence), and refreshed + cluster-wide from that shared DB with perf_sync (see Cluster + sync). What the module does not do is cachedb_local-style + per-operation replication (a cluster_id write streamed between + nodes on every operation) - that would tax the lock-free path + the module exists to keep fast; sharing here is the + pull-from-DB refresh model. Deployments that need per-operation + replication must stay on cachedb_local. + + Data is organized in named collections (hash tables), declared + via the cache_collections parameter. Each cachedb_url points to + one collection; a URL naming no collection uses the collection + named “default”, which always exists. + +Warning + + Work in progress: this module is under active development, but + functionally complete for a single node - data operations, the + background expiry sweep, runtime table growth, statistics, the + huge-page arena, the introspection MI and the observability + events are all in place, whole collections can be persisted to + a db_* backend, and perf_sync refreshes a collection + cluster-wide from that shared DB. Per-operation replication (a + streamed write log between nodes) is intentionally out of scope + - sharing is the pull-from-DB refresh model, not a merge of + divergent copies. + +1.2. Dependencies + +1.2.1. OpenSIPS Modules + + None required. Two are optional: + * clusterer - enables the cluster features: cross-node pull + over its BIN links, the perf_sync peer signal and the + sync_shtag failover hook. Without it the module runs purely + node-local (see Degraded operation). + * clusterer_controller - enables the optional clctr pull + transport (encrypted multicast). Optional at build time + too: a tree built without it simply has no clctr transport, + and everything runs over the clusterer's BIN links. + +1.2.2. External Libraries or Applications + + None. + +1.3. Exported Parameters + +1.3.1. cache_collections (string) + + Declares the collections and, optionally, their initial hash + table size, as a semicolon-separated list of name or name=size + entries. The size is the power-of-2 exponent of the initial + bucket count (as in cachedb_local) and only sets the starting + point - the table grows at runtime as entries accumulate. + Values are clamped to the [4, 24] range; the default is 14 + (16384 buckets). + + The cachedb_local replication marker (“/r”) is rejected: this + cache is single-node. + + Example 1.1. Set cache_collections parameter +... +modparam("cachedb_perf", "cache_collections", "th=16;profiles") +... + +1.3.2. expiry_sweep_period (integer) + + How often, in seconds, expired records are reclaimed. Expired + entries are already invisible to reads the moment they expire - + the sweep only frees their memory, guided by per-bucket hints + so idle collections cost next to nothing. Default is 1 second; + 0 disables the sweep (expired records then hold their memory + until overwritten or deleted). + + Example 1.2. Set expiry_sweep_period parameter +... +modparam("cachedb_perf", "expiry_sweep_period", 5) +... + +1.3.3. cachedb_url (string) + + URL(s) usable from the script or by other modules. The + collection is given by the URL's database part (perf:///name) + or, equivalently, its host part (perf://name) - a host has no + meaning for a local cache, so both forms select the collection. + A URL naming no collection (perf://) uses the “default” + collection. Naming an undefined collection is a startup error. + Multiple URLs may share one collection; use a group + (perf:group_name:///name) to address a specific URL from the + script. + + Example 1.3. Set cachedb_url parameter +... +modparam("cachedb_perf", "cachedb_url", "perf:///th") +modparam("cachedb_perf", "cachedb_url", "perf:prof:///profiles") + +# usage from script: +# cache_store("perf", ...) - collection "th" +# cache_store("perf:prof", ...) - collection "profiles" +... + +1.3.4. growth_load_factor (integer) + + The target number of entries per bucket the maintenance timer + grows the table toward. As entries accumulate the timer splits + buckets to keep the load factor near this value, so lookups + stay flat as the cache scales - the behaviour cachedb_local + lacks. 0 disables growth, leaving the table fixed at its + declared size. Default is 2. + + Example 1.4. Set growth_load_factor parameter +... +modparam("cachedb_perf", "growth_load_factor", 2) +... + +1.3.5. growth_budget (integer) + + The maximum number of bucket splits the maintenance timer + performs on a single run, bounding the work of one growth pass + so the timer never stalls under a burst of inserts. Default is + 4096. + + Example 1.5. Set growth_budget parameter +... +modparam("cachedb_perf", "growth_budget", 4096) +... + +1.3.6. arena_hugepage_mb (integer) + + Size, in megabytes, of a huge-page-backed reservation for the + cache entries. When set, the module reserves this much memory + at startup and backs it with 2 MB pages to cut TLB misses on a + large cache. It does not pick a mechanism: it climbs a + detect-by-trying ladder - overcommit hugetlb pool (MAP_HUGETLB) + then transparent huge pages (MADV_HUGEPAGE) then MADV_COLLAPSE + then plain 4 KB - and keeps the best tier the running kernel + actually grants, which it reports at startup and through the + memory_tier statistic and perf_stats. 0 (default) uses plain + demand-faulted shared memory. + + To make the faster tiers available: allow on-demand huge pages + with sysctl vm.nr_overcommit_hugepages=N (N >= + arena_hugepage_mb/2), and/or enable shmem THP with echo advise + > /sys/kernel/mm/transparent_hugepage/shmem_enabled. Except for + the hugetlb tier (which is unswappable and exempt), the + reservation is mlock-pinned against swap; that needs + LimitMEMLOCK=infinity in the systemd unit, otherwise the module + warns and runs the arena unpinned. + + Example 1.6. Set arena_hugepage_mb parameter +... +modparam("cachedb_perf", "arena_hugepage_mb", 512) +... + +1.3.7. arena_selftest (integer) + + When set to 1, the slab arena runs a self-test at startup and + aborts startup on any mismatch - a permanent, cheap diagnostic. + Default is 0 (off). + + Example 1.7. Set arena_selftest parameter +... +modparam("cachedb_perf", "arena_selftest", 1) +... + +1.3.8. htable_selftest (integer) + + When set to 1, the hash table and its runtime-growth machinery + run a self-test at startup and abort startup on any mismatch. + Default is 0 (off). + + Example 1.8. Set htable_selftest parameter +... +modparam("cachedb_perf", "htable_selftest", 1) +... + +1.3.9. event_expired_collections (string) + + Comma-separated list of the collections that raise + E_CACHEDB_PERF_EXPIRED (one event per reaped key) as the sweep + reclaims them. It is opt-in per collection because a high-churn + collection can reap in bulk, and event delivery is synchronous + - a collection should pay for the per-key events only if + something is listening for them. Empty (default) means no + collection raises the event. See Exported Events. + + Example 1.9. Set event_expired_collections parameter +... +modparam("cachedb_perf", "event_expired_collections", "sessions,subscrip +tions") +... + +1.3.10. db_url (string) + + URL of a db_* (SQL) backend used to persist collections - see + DB persistence. The matching db_* module must be loaded. When + unset, persistence is disabled. The DB is a shared, durable + store; the in-memory cache is a view over it. + + Example 1.10. Set db_url parameter +... +modparam("cachedb_perf", "db_url", "mysql://opensips:pw@localhost/opensi +ps") +... + +1.3.11. db_table (string) + + Table that holds the persisted entries. Default is + “cachedb_perf”. See DB persistence for the schema. + +1.3.12. db_mode (integer) + + Automatic persistence for the collections listed in + persist_collections: 0 = off (default; load/save only on the + perf_load/ perf_save MI commands), 1 = load them from the DB at + startup, 2 = load at startup and save on a graceful shutdown. + + Example 1.11. Set db_mode parameter +... +modparam("cachedb_perf", "db_mode", 2) +... + +1.3.13. persist_collections (string) + + Comma-separated list of the collections that db_mode loads at + startup and saves at shutdown. Empty (default) means none are + persisted automatically - though perf_save/perf_load still work + on any collection on demand. + + Example 1.12. Set persist_collections parameter +... +modparam("cachedb_perf", "persist_collections", "sessions,profiles") +... + +1.3.14. sync_cluster_id (integer) + + Cluster to signal on perf_sync - see Cluster sync. 0 (default) + = off. When set, the clusterer module must be loaded (before + cachedb_perf) and db_url configured; if either is missing, + perf_sync degrades to a DB save with no peer signal (a soft + dependency, never fatal). + + Example 1.13. Set sync_cluster_id parameter +... +loadmodule "clusterer.so" +loadmodule "cachedb_perf.so" +modparam("cachedb_perf", "sync_cluster_id", 1) +... + +1.3.15. sync_shtag (string) + + A clusterer sharing tag, as “name/cluster_id”, that arms the + failover sync. A node whose tag turns active warms every + declared collection from the DB snapshot before the redirected + traffic arrives; a node gracefully demoted to backup saves its + state and signals the peers to reload it - so a failover moves + the cache as one snapshot instead of a storm of misses. + + Requires db_url. The tag only schedules these bulk operations - + lookups are never gated on its state. On a crash failover the + last saved snapshot is the only source, so pair this with + periodic perf_save (or db_mode 2) on the active node. + + Default value is unset (failover sync off). + + Example 1.14. Setting sync_shtag +modparam("clusterer", "sharing_tag", "vip1/1=backup") +modparam("cachedb_perf", "sync_shtag", "vip1/1") + +1.3.16. replicate_collections (string) + + Comma-separated collections whose keys may be fetched from + another node when this one misses (“pull on miss”). Nothing is + pulled unless it is listed here, and the default is to list + nothing. + + The opt-in is deliberate and cannot be inferred: a key is only + worth asking the cluster about if it means the same thing on + every node. That holds for keys derived from the call - the + topology hiding state, for instance - and fails for anything a + script names after something local, where a peer's answer would + be wrong rather than merely useless. Values must be portable + too: a blob that embeds a node's own address is not. + + Requires sync_cluster_id and the clusterer. + + A pulled key is kept: the next request for it is answered + locally and the cluster is never asked again, which is what + makes this a repair rather than a relay. The copy keeps the + expiry the owner had - never a fresh lifetime - so it dies when + the original does instead of outliving it. Note the consequence + for sizing: as traffic spreads, every node tends toward holding + every key, so size the arena for the whole keyspace rather than + its share of it. + + Native counters (created with cache_add) are never served to a + peer. A counter records what happened on the node holding it, + so handing it over would import one node's tally into another; + the requester is told the key is not there, which from its side + is true. + + Default value is unset (no collection is pulled). + + Example 1.15. Setting replicate_collections +modparam("cachedb_perf", "sync_cluster_id", 1) +modparam("cachedb_perf", "replicate_collections", "th") + +1.3.17. pull_transport (string) + + How cross-node pulls travel: bin (default) uses the clusterer's + BIN links. clctr rides the clusterer_controller module's + encrypted multicast plane instead: one datagram reaches every + peer, and the payload is encrypted, which the BIN links are + not. + + The controller is optional, at build time and at run time. When + this build does not include clusterer_controller, or the module + is not loaded, clctr logs a warning and degrades to bin; if the + clusterer module is unavailable too, cross-node pull and sync + are disabled and the cache runs purely node-local. Missing + cluster infrastructure never stops the module from starting. + + Default value is “bin”. + +1.3.18. pull_timeout_ms (integer) + + How long a pull waits for the cluster, in milliseconds + (1..5000). It is a backstop, not the normal cost: with every + peer answering either way, a pull finishes as soon as the last + one has spoken - on a LAN, in a couple of milliseconds. The + timeout only decides how long an unanswered request lingers. + + Default value is “50”. + +1.3.19. pull_negative_ms (integer) + + How long to remember that the whole cluster answered “not here” + for a key, in milliseconds (0..2000; 0 disables it). A SIP + retransmit asks the same question a few hundred milliseconds + later, and without this every retransmit repeats the round of + questions. + + Keep it short. A key may legitimately be created on another + node a moment from now, and a negative that outlives that turns + a transient miss into a hard failure - which is why the + parameter is capped rather than left open. Only a verdict the + whole cluster gave is remembered: a timeout is not absence, and + neither is an answer from a set of nodes that has since + changed. A local write to the key clears it at once. + + Negatives are held outside the cache, so they never appear in + perf_keys or perf_dump and never count as entries. + + Default value is “300”. + +1.3.20. pull_on_miss (integer) + + Repair a miss on the ordinary read path: when a lookup finds + nothing locally, ask the cluster and return whatever comes back + as though it had been here all along. A consumer needs no + changes - cross-node lookups simply start working for the + collections listed in replicate_collections. + + Off by default, and it should stay off on a SIP path for now. + The lookup blocks until the cluster answers or pull_timeout_ms + elapses, and a blocked lookup means a process serving nothing + else in the meantime. A LAN pull takes a couple of milliseconds + and the negative cache absorbs retransmits, but that is a + statement about the common case, not a guarantee under load. + Enable it for maintenance, migration or test paths; a startup + warning repeats this when it is on. + + Default value is “0” (disabled). + +1.4. DB persistence + + With db_url set, a whole collection can be saved to and loaded + from an SQL backend. A save is a full snapshot: the + collection's rows are deleted and every live entry is + re-inserted, with its TTL stored as an absolute wall-clock time + so it survives a restart (already-expired entries are skipped + on both save and load). Native counters round-trip as their + decimal value. + + Trigger it on demand with the perf_save / perf_load MI + commands, or automatically via db_mode. This is single-node + durability; for cross-node sharing over the same DB, see + Cluster sync (still not per-operation replication). + +Warning + + A save/load is a full, blocking snapshot of the collection: one + SQL statement per entry, run synchronously in the process that + issued it. On a large collection (this module is built for + millions of entries) or a slow backend such as db_text or + db_sqlite, that can take a long time and stall that process for + its duration. Treat it as a maintenance / bootstrap operation - + startup warm-up, shutdown flush, an occasional snapshot or a + perf_sync refresh - never on a per-request path and not on a + tight timer. Frequent whole-collection persistence is an + anti-pattern; if you need durable per-key writes on every + operation, this is the wrong tool. + + The table (default “cachedb_perf”) needs these columns: +collection string - the collection name +pkey string - the cache key +pvalue binary - the value (BLOB; binary-safe) +expires int - absolute unix expiry, 0 = never + +1.5. Cluster sync + + Built on the same DB: with sync_cluster_id set, perf_sync (MI + command and script function) saves a collection to the DB and + then signals every node in the cluster to reload it from there. + The signal is one small message per sync - not per cache + operation - so it costs nothing on the hot path. + +Warning + + A reload overwrites a peer's copy from the DB, so perf_sync is + for single-writer / read-replica topologies: one node (or the + application writing the DB directly) is the authority for a + collection, the others refresh from it. A node that also takes + its own local writes would lose the unsaved ones on a reload - + this is convergence to a shared source of truth, not a merge of + divergent copies, and deliberately not per-operation + replication. + + A node that reloads because of a peer's perf_sync raises + E_CACHEDB_PERF_SYNCED. With no clusterer or sync_cluster_id 0, + perf_sync still saves to the DB, just without the peer signal. + + When active, the module's capability shows in the clusterer's + clusterer_list_cap MI command as “cachedb-perf-sync”. Note that + the state the clusterer reports there (“Ok”) only means the + capability is registered and enabled: this module does not take + part in the clusterer's startup data-sync, so that field never + reflects whether the caches have converged. It would read “not + synced” only if the capability were disabled administratively. + To see the sync activity itself, use the last_sync_out / + last_sync_in / last_sync_source fields of perf_stats, which + report how many seconds ago this node last pushed a snapshot + and last reloaded one at a peer's request (-1 = never), or + subscribe to E_CACHEDB_PERF_SYNCED. Between syncs the nodes are + expected to differ - convergence is on demand, by design. + + Because it runs a full save first, perf_sync carries the same + blocking cost as perf_save (see the warning under DB + persistence): it is an occasional refresh, not something to + fire on a timer or per request. + + Example 1.16. Cluster-sync setup (one authority, two read + replicas) +# on every node - clusterer must load before cachedb_perf (the module +# declares a soft dependency, so init order is handled either way): +loadmodule "clusterer.so" +modparam("clusterer", "my_node_id", 1) # 2 and 3 on the other n +odes +... +loadmodule "cachedb_perf.so" +modparam("cachedb_perf", "cache_collections", "profiles") +modparam("cachedb_perf", "db_url", "mysql://opensips:pw@dbhost/opensips" +) +modparam("cachedb_perf", "sync_cluster_id", 1) + +# on the authority node, after it has updated the "profiles" collection: +# opensips-cli -x mi cachedb_perf:perf_sync profiles +# -> saves "profiles" to the DB, signals nodes 2 and 3 to reload it +# +# or from script (e.g. after a reload route), same save-then-broadcast: +# perf_sync("profiles"); +# +# the replicas raise E_CACHEDB_PERF_SYNCED when they finish reloading: +event_route[E_CACHEDB_PERF_SYNCED] { + xlog("L_INFO", "reloaded $param(collection) from node $param(source_ +node)\n"); +} + +1.6. Degraded operation: what each surface does without its modules + + The cache itself never depends on the cluster plane. Whatever + is missing, the module starts, serves node-local traffic, and + says at startup exactly what it turned off. There are two + degraded modes worth knowing precisely; every output below is + captured from a live instance, not paraphrased. + + Mode 1 - clusterer loaded, clusterer_controller absent. The + only thing lost is the clctr transport; pull and sync are fully + functional over the clusterer's BIN links. One warning at + startup, worded for the reason the controller is missing: +# controller not in this build: +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but this + build carries no clusterer_controller support - falling + back to 'bin' +# controller in the build but not loaded: +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but + clusterer_controller is not loaded - falling back to 'bin' + + Mode 2 - neither module available. Pull and sync are disabled + entirely; the cache runs purely node-local. Three warnings at + startup: +WARNING:cachedb_perf:mod_init: clusterer module not available - + the cluster features are disabled; load clusterer before + cachedb_perf +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but this + build carries no clusterer_controller support - falling + back to 'bin' +WARNING:cachedb_perf:mod_init: replicate_collections is set but + the cluster is not available (needs sync_cluster_id + + clusterer) - cross-node pull disabled + + Unaffected in both modes: every local surface. The cachedb + script API (cache_store / cache_fetch / cache_add / cache_sub / + cache_remove on "perf"), the glob functions (perf_del, + perf_mget, perf_mget_json), the local MI set (perf_get / + perf_set / perf_probe / perf_keys / perf_scan / perf_dump / + perf_ttl / perf_del / perf_stats / perf_stats_reset), DB + persistence (perf_save / perf_load, db_mode), and the four + local events (E_CACHEDB_PERF_EXPIRED / NOMEM / GROWN / + MEM_DEGRADED). + + perf_pull - in mode 1 the pull runs over bin; on a single node + (or when no peer holds the key) it reports where the answer did + not come from. In mode 2 it refuses: +# mode 1: +{ "source": "no-answer" } + +# mode 2: +{ "code": 500, + "message": "cross-node pull not active + (replicate_collections)" } + + perf_cluster_probe - mode 1 probes over bin (on a lone node: + error 500, "could not start the probe - no peers, or no free + pull slot"); mode 2 answers error 400, "cross-node pull is not + active for this collection (replicate_collections)". + + perf_sync (MI and script function) - never fails for cluster + reasons. Mode 1 saves and signals the peers; mode 2 degrades to + the DB save alone and says so: +# mode 1: +{ "collections": 2, "saved": 1, "broadcast": 2 } + +# mode 2: +{ "collections": 2, "saved": 1, "broadcast": 0, + "note": "cluster sync inactive (no clusterer / + cluster_id 0) - saved to the DB only" } + + perf_stats - the per-collection and memory sections are + identical in both modes (including the pulled_from_cluster / + served_to_cluster counters, which simply stay 0). The + difference is the cluster object: present in mode 1 (ids, + membership, the pull counters and slot count, and a topology + array), absent in mode 2. +# mode 1 only: +"cluster": { + "cluster_id": 1, "my_node_id": 1, "peers_up": 0, + ... + "pull_slots": 64, + "topology": [ { "node_id": 1, "role": "self", "membership": "up" } ] +} + + cache_fetch with pull_on_miss=1 - mode 1: a miss on an opted-in + collection blocks up to pull_timeout_ms asking the cluster, + exactly as documented under pull_on_miss. Mode 2: no pull + exists, so a miss is a plain immediate miss - no blocking, no + timeout, no negative cache. + + E_CACHEDB_PERF_SYNCED - only ever raised when a peer's sync + signal arrives, so it keeps firing in mode 1 and can never fire + in mode 2. Subscribing to it costs nothing either way. + +1.7. Exported Statistics + + All counters are aggregated per-process and summed only when + read, so instrumentation never touches a shared cache line on + the hot path. Query with get_statistics cachedb_perf:. + * hits / misses - fetch outcomes (expired counts as a miss) + * stores / removes - write and explicit delete operations + (removes counts only remove/perf_del, never TTL expiry) + * expired - records reclaimed by the TTL expiry sweep (this + is where timed-out keys are accounted, separate from + removes) + * destroyed - total records whose cells were freed back to + the arena, from any cause; it equals removes + expired. + Overwriting an existing key does not count here (the cell + is reused in place), so entries = created - destroyed. A + large gap between stores and entries with a small destroyed + means most churn is same-key overwrites rather than expiry + or deletes. + * entries - live records across all collections + * seqlock_retries - optimistic-read retries (the contention + signal) + * lock_fallbacks - reads that fell back to the bucket lock + * arena_bytes / arena_chunks - memory taken from shm + * memory_tier_probe / memory_tier_active - the huge-page tier + the host was probed for, and the one the module's own arena + actually runs on (1 hugetlb .. 4 plain 4K). They differ + when arena_hugepage_mb could not be satisfied. + * hugepage_arena_active, hugepage_arena_total_bytes, + hugepage_arena_used_bytes, hugepage_arena_free_bytes - the + module's own hugepage arena, which is SEPARATE from the + OpenSIPS shm arena. Records live in one or the other, never + both, so these do not add to arena_bytes. + + Cross-node pull statistics. These exist only when pull_on_miss + is enabled. They are grouped so that two identities hold, which + is the point of having them: every miss is accounted on the way + out, and every request is accounted on the way back. A missing + counter here is not cosmetic - it is a miss that vanished. + * pulls_requested - misses that turned into a request on the + wire. pulls_served - requests from peers this node + answered. + * pulls_suppressed - asks absorbed by an already-cached + negative (pull_negative_ms); the second and later asker for + a key a peer has already denied. + * pulls_skip_notreplicated, pulls_skip_toolong, + pulls_skip_nopeers, pulls_skip_noslot - misses refused at + the gate before anything was sent. They are kept apart + because each calls for a different action: pull is off for + that collection; the key or collection name is too long to + ask for at all; no live cluster member to ask; or the slot + table is full and nothing was evictable. + * pulls_received - a value came back. pulls_negative - a peer + answered that it does not have the key. pulls_oversize - a + peer has it but it exceeds the cluster transport limit. + pulls_timed_out - nothing came back in pull_timeout_ms. + pulls_send_failed - the transport refused the datagram, so + no peer was ever asked. + * pulls_stored - answers written into the cache. + pulls_in_flight - requests outstanding right now (a gauge, + not a total). + * pulls_orphaned - a waiter gave up but the slot was kept in + case the answer still arrives, for pull_linger_ms. Its + outcomes: pulls_late_stored (arrived and was stored - + convergence that would otherwise be lost), + pulls_late_superseded (a local write had already filled the + key), pulls_late_expired (arrived past the linger and was + refused as stale), pulls_orphan_expired (no late answer + ever came - the ordinary end of a timeout), + pulls_orphan_evicted (the slot was reclaimed early because + the pool ran dry). + * pulls_abandoned - slots the reaper released because the + caller never collected them. Distinct from a timeout, which + the caller DID collect: a non-zero value here is a defect + signal, not tuning. + * pulls_foreign_cluster - pull messages that arrived on a + controller cluster this module does not sync on and were + refused. Non-zero means either a genuine multi-cluster node + behaving correctly, or sync_cluster_id naming a cluster the + controller does not manage; the accompanying warning tells + the two apart. + + The two identities: + +misses = pulls_requested + pulls_suppressed + + pulls_skip_notreplicated + pulls_skip_toolong + + pulls_skip_nopeers + pulls_skip_noslot + +pulls_requested = pulls_received + pulls_negative + pulls_oversize + + pulls_timed_out + pulls_send_failed + pulls_in_flight + + The second holds exactly on a two-node cluster. pulls_negative + is counted per REPLY, so with more peers one request can raise + it more than once. + + The perf_stats MI command gives the same figures broken down + per collection, plus load factor, overflow occupancy, + retries-per-1k-reads, the memory-backing description and + hit_rate_pct - hits / (hits + misses) as a percentage. On a + healthy server the large majority of lookups hit (upwards of + 80% under steady dialog traffic); a persistently low or falling + hit rate means the cached state is being lost or is expiring + before it is used. The same guidance rides inline in the + hit_rate_note field. + +1.8. Exported MI Functions + +1.8.1. perf_stats + + Reports per-collection statistics (entries, buckets, overflow, + hits/misses/stores/removes, load factor, seqlock retries and + retries-per-1k-reads) plus the arena occupancy and the achieved + memory tier. With no parameter it reports every collection; an + optional collection name restricts it to one. + + Name: perf_stats. Parameters: collection (optional). + + Example 1.17. perf_stats usage +opensips-cli -x mi cachedb_perf:perf_stats +opensips-cli -x mi cachedb_perf:perf_stats th + +1.8.2. perf_stats_reset + + The counters behind perf_stats - hits, misses, stores, removes, + expired, destroyed, retries - are running totals since startup, + so every rate derived from them is a lifetime average. A burst + of misses right after a restart, when sequential requests + arrive for dialogs older than the cache, keeps dragging the hit + rate down long after the cache has recovered. This command + re-baselines them so the next reading covers a fresh interval, + without restarting OpenSIPS. + + The counters themselves are not rewound: each process owns its + own counter cache line and must never have it written from + another process. Only a baseline is recorded, and the reported + figures are the difference. Live gauges - entries, buckets, + overflow, load factor and the arena figures - are read from + current state rather than from the counters, so a reset does + not disturb them. + + With no parameter every collection is reset; an optional + collection name restricts it to one. + + Name: perf_stats_reset. Parameters: collection (optional). + + Example 1.18. perf_stats_reset usage +opensips-cli -x mi cachedb_perf:perf_stats_reset +opensips-cli -x mi cachedb_perf:perf_stats_reset th + +1.8.3. Key introspection: perf_keys / perf_scan / perf_dump / +perf_get / perf_probe / perf_pull / perf_set / perf_ttl / perf_del + + These give an operator the visibility that cachedb_local lacks. + All are lock-free: the walkers take no bucket locks (seqlock + reads), so unlike a cachedb_local key scan they never stall SIP + traffic. Every command carries the perf_ prefix, matching the + script functions and staying clear of the core's bare get/set. + The optional collection selects the table; omitted, it is the + groupless cachedb_url's collection. + * perf_keys [collection] [limit] - names (and TTL) of + the keys matching a shell glob, bounded (default 1000; the + reply carries a note when it truncates). The KEYS + equivalent. + * perf_scan [glob] [count] - cursored incremental + iteration with Redis SCAN semantics over the default + collection: start with cursor 0 and repeat with the + returned cursor until it comes back 0. An entry present + throughout is returned at least once; count bounds the + buckets visited per call. This is the answer for a large + cache, where perf_keys would truncate. + * perf_dump [collection] [limit] - like perf_keys but + includes the values; values are opt-in, never the default. + * perf_get [collection] - one key: its value, remaining + TTL (-1 = never) and size. + * perf_pull [collection] - fetch one key from the + cluster, for a collection listed in replicate_collections. + Reports where the answer came from: local (this node had it + after all), cluster (a peer had it, with its remaining + TTL), absent (every peer answered, none has it) or + no-answer (nobody answered in time, or there was nobody to + ask). The last two are deliberately different: absence is a + fact only when the whole cluster has said so. + * perf_probe [collection] - is the key here: its size + and remaining TTL, but never the value. Not merely a + cheaper perf_get - it shares the whole read path (same + optimistic loop, lock fallback and expiry rules) and stops + before the copy-out, so it cannot disagree with a read + about whether a key is present; it allocates nothing and + never touches the record's payload. Use it to answer “do + you have this key?”, where a read would pay for bytes + nobody wants. + * perf_set [ttl] [collection] - write one key; + ttl is seconds (0 or omitted = never expires). + * perf_ttl [collection] - re-arm the TTL of + every key matching the glob without rewriting its value + (one atomic expiry store under the bucket lock, so + lock-free readers are undisturbed); ttl is seconds (0 = + never). Returns the count updated; a literal key matches + exactly one. + * perf_del [collection] - delete every key matching + the glob; returns the count. The MI face of the perf_del() + script function. + + Example 1.19. introspection usage +opensips-cli -x mi cachedb_perf:perf_keys "session-*" +opensips-cli -x mi cachedb_perf:perf_keys "session-*" th 50 +opensips-cli -x mi cachedb_perf:perf_scan 0 +opensips-cli -x mi cachedb_perf:perf_scan 384 "user-*" 128 +opensips-cli -x mi cachedb_perf:perf_dump "profile-*" +opensips-cli -x mi cachedb_perf:perf_get session-abc123 +opensips-cli -x mi cachedb_perf:perf_set greeting hello 300 +opensips-cli -x mi cachedb_perf:perf_ttl "session-*" 1800 +opensips-cli -x mi cachedb_perf:perf_del "session-abc*" + +1.8.4. perf_save / perf_load + + Persist a collection to, or restore it from, the db_url backend + (see DB persistence). With no argument they operate on every + declared collection; with a collection name, only that one. The + reply reports how many collections and entries were written or + read. + + Example 1.20. perf_save / perf_load usage +opensips-cli -x mi cachedb_perf:perf_save +opensips-cli -x mi cachedb_perf:perf_save sessions +opensips-cli -x mi cachedb_perf:perf_load sessions + +1.8.5. perf_sync + + Save a collection to the DB and signal the cluster to reload it + (see Cluster sync); all declared collections if none is named. + Also available as a script function, perf_sync([collection]). + + Example 1.21. perf_sync usage +opensips-cli -x mi cachedb_perf:perf_sync sessions + +1.9. Exported Events + + Every event is gated by evi_probe_event(), so with no + subscriber it costs a single shared read and nothing more; none + of them sit on the lock-free get/set path. + +1.9.1. E_CACHEDB_PERF_EXPIRED + + Raised by the sweep for each expired record it reclaims, but + only for the collections named in event_expired_collections + (opt-in, since a high-churn collection reaps in bulk and + delivery is synchronous). Parameters: collection, key. + +1.9.2. E_CACHEDB_PERF_NOMEM + + Raised when a write is dropped because the arena is full - the + cache is out of memory and rejecting stores. One event per + dropped write (subscribers should expect bursts under memory + pressure). Parameters: collection, key, size (the value's byte + length). + +1.9.3. E_CACHEDB_PERF_GROWN + + Raised by the maintenance timer after it grows a collection's + table. Parameters: collection, prev_buckets, buckets, splits, + entries. + +1.9.4. E_CACHEDB_PERF_MEM_DEGRADED + + Raised once at startup when arena_hugepage_mb was set but the + arena settled on a tier below hugetlb (missing + vm.nr_overcommit_hugepages, for instance) - the node is running + slower than intended. Parameters: requested_mb, tier (1 hugetlb + .. 4 plain 4K), backing (its description), overcommit_pages. + +1.9.5. E_CACHEDB_PERF_SYNCED + + Raised on a node that reloaded a collection from the DB because + a peer issued perf_sync (see Cluster sync). Parameters: + collection, source_node (the cluster id of the node that issued + the sync). + +1.10. Exported Functions + + Single-key operations go through the core cache functions + (cache_store(), cache_fetch(), ...) with the “perf” backend. + The functions below are the module's own glob (multi-key) + operations. All of them match keys with shell-style globs + (fnmatch), walk the table lock-free and give the Redis SCAN + class of guarantee: an entry mutated concurrently may be seen + once, twice or not at all. Unlike cachedb_local's + cache_remove_chunk(), these are perf_-prefixed - scripts + migrating from cachedb_local must rename those calls. When the + optional collection argument is omitted, they operate on the + collection of the default (groupless) cachedb_url - exactly + where cache_store("perf", ...) writes. + +1.10.1. perf_del(glob[, collection]) + + Deletes every key matching the glob (expired entries included). + Returns the number of keys removed, or -1 (false) if none + matched. + + Example 1.22. perf_del() usage +... +perf_del("session-*"); +perf_del("th-*", "th"); +... + +1.10.2. perf_mget(glob, keys_avp, vals_avp[, collection[, limit]]) + + Returns every live key/value pair matching the glob into two + writable variables (use AVPs - each match adds one value to + each, and the indexes correspond pairwise; ordering is + unspecified). limit bounds the number of matches, default 1000, + 0 = unbounded. Returns the match count, or -1 (false) if none + matched. + + Example 1.23. perf_mget() usage +... +if (perf_mget("user-*", $avp(k), $avp(v))) { + xlog("first match: $(avp(k)[0]) = $(avp(v)[0])\n"); +} +... + +1.10.3. perf_mget_json(glob, dst_var[, collection[, limit]]) + + Like perf_mget(), but returns all matches as one JSON object + {"key":"value",...} in a single writable variable ({} when + nothing matches). Quote, backslash and control bytes are + escaped, so binary values survive; bytes above 0x7F pass + through unescaped - strict JSON consumers therefore need UTF-8 + values. Returns the match count, or -1 (false) if none matched. + + Example 1.24. perf_mget_json() usage +... +if (perf_mget_json("user-*", $var(blob), , 100)) + xlog("users: $var(blob)\n"); +... + + Documentation Copyrights: + + Copyright © 2026 Yury Kirsanov diff --git a/modules/cachedb_perf/bench/.gitignore b/modules/cachedb_perf/bench/.gitignore new file mode 100644 index 00000000000..c68678bfe11 --- /dev/null +++ b/modules/cachedb_perf/bench/.gitignore @@ -0,0 +1,13 @@ +hashtest +lookup +structs +worker +expire2 +concur +wbuf +queue +warmup +hugetlb +hugetlb2 +mlockt +rpath diff --git a/modules/cachedb_perf/bench/Makefile b/modules/cachedb_perf/bench/Makefile new file mode 100644 index 00000000000..845e649a9c7 --- /dev/null +++ b/modules/cachedb_perf/bench/Makefile @@ -0,0 +1,34 @@ +# cachedb_perf design benchmarks - standalone, no OpenSIPS build needed. +# Each program embeds core_hash() verbatim from ../../../hash_func.h. + +CC ?= gcc +CFLAGS ?= -O2 -Wall +PROGS = hashtest lookup structs worker expire2 concur wbuf queue warmup hugetlb hugetlb2 mlockt rpath + +all: $(PROGS) + +%: %.c + $(CC) $(CFLAGS) -o $@ $< + +concur wbuf queue rpath: %: %.c + $(CC) $(CFLAGS) -pthread -o $@ $< + +run: all + @echo "=== 1. hash distribution: is core_hash to blame? ==="; ./hashtest + @echo; echo "=== 2. lookup cost vs load factor ==="; ./lookup + @echo; echo "=== 3. index structure shootout ==="; ./structs + @echo; echo "=== 4. deferred sorting / worker ==="; ./worker + @echo; echo "=== 5. expiry strategies ==="; ./expire2 + @echo; echo "=== 6. concurrent read path (threads) ==="; ./concur + @echo; echo "=== 7. write staging buffer ==="; ./wbuf + @echo; echo "=== 8. queued (producer/consumer) writes ==="; ./queue + @echo; echo "=== 9. first-touch cost ==="; ./warmup + @echo; echo "=== 10. huge pages (needs: sysctl -w vm.nr_hugepages=160) ==="; ./hugetlb + @echo; echo "=== 11. modern routes: base/collapse/madvise/hugetlb/huge1g ==="; ./hugetlb2 base + @echo; echo "=== 12. swap pinning (mlock) ==="; ./mlockt + @echo; echo "=== 13. read-path protocols: seqlock / hybrid / qsbr ==="; ./rpath + +clean: + rm -f $(PROGS) + +.PHONY: all run clean diff --git a/modules/cachedb_perf/bench/README.md b/modules/cachedb_perf/bench/README.md new file mode 100644 index 00000000000..8590eacddfa --- /dev/null +++ b/modules/cachedb_perf/bench/README.md @@ -0,0 +1,60 @@ +# cachedb_perf design benchmarks + +Standalone reproductions of every figure in `../DESIGN.md`. No OpenSIPS build +required — each program embeds `core_hash()` verbatim from `hash_func.h` and +models the data layouts directly. + +```bash +make && make run +``` + +Common workload: **50 000 keys**, 16-byte hex keys shaped like `th_store` +thids, 200-byte values, and allocations deliberately interleaved with junk +allocations so entries are scattered the way `shm_malloc` leaves them after a +busy run. Measurements are of **successful** point lookups — the hot path — +not misses. + +| program | question it answers | +|---|---| +| `hashtest` | Is `core_hash()` a bad hash? (chi², empty buckets, max chain vs FNV-1a) | +| `lookup` | What does load factor cost? (512 vs 65536 buckets) | +| `structs` | Chained vs sorted-array vs cache-line bucket vs flat open addressing | +| `worker` | Does moving the sort off the hot path pay? (eager vs deferred + merge cost) | +| `expire2` | Full sweep vs per-bucket min-expires hint vs timer wheel | +| `concur` | Does the lock-on-every-read path limit scaling? (1/2/4/8 threads) | +| `wbuf` | Does a write-staging buffer help? (shared vs per-process, + read penalty) | +| `queue` | Does a queued producer/consumer write path help? (fixed thread budget) | +| `warmup` | What does first-touch page faulting cost, and does pre-warming help? | +| `hugetlb` | Do 2M huge pages help? Needs `sysctl -w vm.nr_hugepages=160` first | +| `hugetlb2` | Modern routes: `MADV_COLLAPSE`, THP-shmem, overcommit pool, 1GB pages. Takes a mode arg (`base`/`collapse`/`madvise`/`hugetlb`/`huge1g`) and verifies pages went huge via meminfo | +| `mlockt` | Can the arena be pinned against swap? mlock cost, fork inheritance, meminfo verification | +| `rpath` | Is the seqlock the fastest read protocol? seqlock vs versionless-bump hybrid vs QSBR pointer-publication, with seqlock retries/1k reads (DESIGN 2.7) | + +## Caveats + +These are **models, not the module**. They measure structure and cache +behaviour in a single process; they do not model shm allocation, multi-process +coherence, or OpenSIPS locking primitives. Treat the numbers as ranking +designs, not predicting throughput. + +`concur.c` and `rpath.c` are the threaded ones, and both results went +*against* the hypothesis each was written to test. `concur.c` **refuted** +lock-on-every-read as a scaling killer: `cachedb_local` scales 8.4× on 8 +threads, because with 65 536 buckets workers rarely collide on a bucket lock. +The proposed design's 3–4× is a per-operation constant factor — no atomic RMW +on reads, one cache line per bucket, tag filtering — not a scaling win. Do +not quote it as one. Note also that it gives `cachedb_local` its best case, a +perfectly sized table; against the shipped 512-bucket default the gap is +~90×. `rpath.c` showed dropping the seqlock for QSBR pointer-publication +reads is worth nothing at 100% reads — the version loads are free on x86/TSO +— and pays only under single-hot-bucket write contention SIP traffic does not +exhibit; only the versionless TTL bump survived into the design (CP-04). +Note `concur.c`'s writers only bump versions (no slot churn), and its +`tag | 1` mapping halves the tag alphabet — both fine for what it measures, +neither to be copied into the module. + +`expire2.c` supersedes an earlier `expire.c` that was unsound: its +min-expires hint was reset to a value that defeated skipping, its wheel loop +was dead-code-eliminated because the counter was never printed, and its +expiry spread put ~50% of entries due per sweep rather than a realistic ~0.03%. +Do not resurrect it. diff --git a/modules/cachedb_perf/bench/cdbstress.c b/modules/cachedb_perf/bench/cdbstress.c new file mode 100644 index 00000000000..10960e18f72 --- /dev/null +++ b/modules/cachedb_perf/bench/cdbstress.c @@ -0,0 +1,222 @@ +/* + * CP-16 multi-process correctness soak for cachedb_perf. Throwaway - NOT + * for the PR. W worker PROCESSES hammer one backend with a get/set/remove/ + * add mix while the maintenance timer splits buckets underneath them, then + * four invariants are checked: + * + * 1. no torn read - every value is written all-bytes-equal; any hit read + * back with mixed bytes means a reader saw a half-done + * write (a seqlock failure). + * 2. no lost update - N adds of +1 across all workers must equal the sum of + * the counter values (add's RMW under the bucket lock). + * 3. no lost key - "immortal" keys inserted once and never removed must + * be found, with the right value, all through the run + * and after all the splits (a split must not drop one). + * 4. no crash - the whole thing runs to completion (run under a + * redzone allocator to also catch use-after-free). + */ +#include +#include +#include +#include +#include "../../sr_module.h" +#include "../../dprint.h" +#include "../../str.h" +#include "../../mem/mem.h" +#include "../../mem/shm_mem.h" +#include "../../cachedb/cachedb.h" + +static int mod_init(void); + +static char *stress_url = NULL; +static int n_workers = 8; +static int n_ops = 2000000; /* per worker */ +static int n_imm = 5000; /* immortal keys (read-only after init) */ +static int n_chn = 5000; /* churn keys (set/remove) */ +static int n_ctr = 64; /* counters (add +1) */ +static int val_sz = 200; + +#define MAXW 64 +struct ctrl { + volatile int ready, done; + unsigned long adds[MAXW]; /* per-worker +1 count on counters */ + unsigned long torn[MAXW]; /* mixed-byte reads seen */ + unsigned long imm_lost[MAXW]; /* immortal miss/wrong-value */ + unsigned long gets[MAXW], sets[MAXW], rems[MAXW]; +}; +static struct ctrl *C; +static str url_s; +static cachedb_funcs cdbf; + +static const param_export_t params[] = { + {"url", STR_PARAM, &stress_url}, + {"n_workers", INT_PARAM, &n_workers}, + {"n_ops", INT_PARAM, &n_ops}, + {"n_imm", INT_PARAM, &n_imm}, + {"n_chn", INT_PARAM, &n_chn}, + {"n_ctr", INT_PARAM, &n_ctr}, + {0,0,0} +}; + +/* key layout: [0,n_imm) immortal | [n_imm,n_imm+n_chn) churn | rest counters */ +static void mkkey(char *buf, int idx) +{ + if (idx < n_imm) sprintf(buf, "imm-%08x", idx); + else if (idx < n_imm+n_chn) sprintf(buf, "chn-%08x", idx); + else sprintf(buf, "ctr-%08x", idx); +} + +/* a hit value must be all-bytes-equal (that is how every set writes it); + * returns 1 if consistent, 0 if torn */ +static int val_ok(const str *v) +{ + int i; + if (v->len == 0) + return 1; + for (i = 1; i < v->len; i++) + if (v->s[i] != v->s[0]) + return 0; + return 1; +} + +static void stress_worker(int no) +{ + cachedb_con *con; + unsigned seed = 12345 + no * 7919; + char kb[32], vbuf[8192]; + str a, v; + int i; + + if (no >= MAXW) return; + con = cdbf.init(&url_s); + if (!con) { LM_ERR("worker %d init failed\n", no); return; } + + __sync_add_and_fetch(&C->ready, 1); + while (C->ready < n_workers) usleep(200); + + for (i = 0; i < n_ops; i++) { + seed = seed * 1103515245u + 12345u; + unsigned r = seed >> 8; + int cls = r % 100; + if (cls < 45) { + /* GET + verify (immortal or churn) */ + int idx = (n_imm + n_chn) ? r % (n_imm + n_chn) : 0; + mkkey(kb, idx); a.s = kb; a.len = strlen(kb); + v.s = NULL; v.len = 0; + cdbf.get(con, &a, &v); + C->gets[no]++; + if (v.s) { + if (!val_ok(&v)) + C->torn[no]++; + pkg_free(v.s); + } else if (idx < n_imm) { + C->imm_lost[no]++; /* immortal must always exist */ + } + } else if (cls < 70) { + /* SET a churn key, all-bytes-equal */ + int idx = n_chn ? n_imm + (r % n_chn) : 0; + mkkey(kb, idx); a.s = kb; a.len = strlen(kb); + memset(vbuf, (int)(seed & 0xFF), val_sz); + v.s = vbuf; v.len = val_sz; + cdbf.set(con, &a, &v, 0); + C->sets[no]++; + } else if (cls < 80) { + /* REMOVE a churn key (churn -> arena reuse + relink) */ + int idx = n_chn ? n_imm + (r % n_chn) : 0; + mkkey(kb, idx); a.s = kb; a.len = strlen(kb); + cdbf.remove(con, &a); + C->rems[no]++; + } else { + /* ADD +1 to a counter */ + int idx = n_imm + n_chn + (n_ctr ? r % n_ctr : 0); + int nv = 0; + mkkey(kb, idx); a.s = kb; a.len = strlen(kb); + if (cdbf.add(con, &a, 1, 0, &nv) == 0) + C->adds[no]++; + } + } + + if (__sync_add_and_fetch(&C->done, 1) == n_workers) { + unsigned long adds=0, torn=0, ilost=0, gets=0, sets=0, rems=0; + unsigned long csum=0, imiss=0, iwrong=0; + int j; + for (j=0;jadds[j]; torn+=C->torn[j]; + ilost+=C->imm_lost[j]; gets+=C->gets[j]; sets+=C->sets[j]; + rems+=C->rems[j]; } + /* verify counters: sum of values == sum of successful adds */ + for (j = 0; j < n_ctr; j++) { + char b[32]; str ka, cv; long long cval; + mkkey(b, n_imm + n_chn + j); ka.s=b; ka.len=strlen(b); + cv.s=NULL; cv.len=0; + if (cdbf.get(con, &ka, &cv) == 0 && cv.s) { + cval = strtoll(cv.s, NULL, 10); csum += cval; + pkg_free(cv.s); + } + } + /* verify every immortal is present with its exact value */ + for (j = 0; j < n_imm; j++) { + char b[32]; str ka, iv; int q; + mkkey(b, j); ka.s=b; ka.len=strlen(b); + iv.s=NULL; iv.len=0; + if (cdbf.get(con, &ka, &iv) != 0 || !iv.s) { imiss++; continue; } + for (q = 0; q < iv.len; q++) + if ((unsigned char)iv.s[q] != (unsigned char)(j & 0xFF)) + { iwrong++; break; } + pkg_free(iv.s); + } + LM_NOTICE("CP16 STRESS: workers=%d ops/w=%d gets=%lu sets=%lu " + "rems=%lu adds=%lu\n", n_workers, n_ops, gets, sets, rems, adds); + LM_NOTICE("CP16 RESULT: torn_reads=%lu | counters sum=%lu vs " + "adds=%lu (%s) | immortals miss=%lu wrong=%lu (%s) => %s\n", + torn, csum, adds, csum==adds?"OK":"LOST-UPDATE", + imiss, iwrong, (imiss==0&&iwrong==0)?"OK":"LOST-KEY", + (torn==0 && csum==adds && imiss==0 && iwrong==0) ? + "PASS" : "*** FAIL ***"); + } + while (1) sleep(60); +} + +static proc_export_t procs[] = { + {"cdbstress worker", 0, 0, stress_worker, 0, PROC_FLAG_INITCHILD}, + {0,0,0,0,0,0} +}; + +struct module_exports exports = { + "cdbstress", MOD_TYPE_DEFAULT, MODULE_VERSION, DEFAULT_DLFLAGS, + 0, 0, 0, 0, params, 0, 0, 0, 0, procs, + 0, mod_init, (response_function)0, 0, 0, 0 +}; + +static int mod_init(void) +{ + cachedb_con *con; + char kb[32], *val; + str a, v; + int i; + + if (!stress_url) { LM_ERR("url required\n"); return -1; } + url_s.s = stress_url; url_s.len = strlen(stress_url); + procs[0].no = n_workers; + + if (cachedb_bind_mod(&url_s, &cdbf) < 0) { + LM_ERR("cannot bind %s\n", stress_url); return -1; } + if (!cdbf.get||!cdbf.set||!cdbf.remove||!cdbf.add) { + LM_ERR("backend lacks ops\n"); return -1; } + + C = shm_malloc(sizeof *C); memset(C, 0, sizeof *C); + + con = cdbf.init(&url_s); + if (!con) { LM_ERR("init failed\n"); return -1; } + val = pkg_malloc(val_sz); + + /* immortals: value = all bytes (i & 0xFF), never touched again */ + for (i = 0; i < n_imm; i++) { + mkkey(kb, i); a.s=kb; a.len=strlen(kb); + memset(val, i & 0xFF, val_sz); v.s=val; v.len=val_sz; + cdbf.set(con, &a, &v, 0); + } + pkg_free(val); + LM_NOTICE("cdbstress: seeded %d immortals; %d workers x %d ops " + "(growth runs concurrently)\n", n_imm, n_workers, n_ops); + return 0; +} diff --git a/modules/cachedb_perf/bench/concur.c b/modules/cachedb_perf/bench/concur.c new file mode 100644 index 00000000000..c98372e367a --- /dev/null +++ b/modules/cachedb_perf/bench/concur.c @@ -0,0 +1,193 @@ +/* + * The claim under test: cachedb_local's read path takes a WRITE lock on every + * fetch, so N workers reading disjoint keys still ping-pong bucket cache lines. + * + * A current : chained bucket, spinlock acquired on every read + * B proposed : 64-byte bucket, 1-byte tags, seqlock optimistic read + * (readers never write -> lines stay Shared) + * + * Scaled 1..8 threads, 100% read and 95/5 read/write. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned core_hash(const str *s1,const str *s2,const unsigned size) +{ char *p,*end; register unsigned v; register unsigned h=0; + end=s1->s+s1->len; + for(p=s1->s;p<=(end-4);p+=4){v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3];ch_h_inc;} + v=0; for(;p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; } + +#define NKEYS 50000 +#define VALLEN 200 +#define KLEN 16 +#define SECS 2 + +/* ---------------- A: current ---------------- */ +typedef struct centry { str attr, value; unsigned expires, ttl; int synced; + struct centry *next; } centry; +typedef struct { centry *e; volatile int lock; } abucket; +#define ANB 65536 +static abucket *A; + +/* ---------------- B: cache-line bucket, tags, seqlock ---------------- */ +#define BSLOTS 6 +#define BNB 16384 +typedef struct { unsigned short klen; unsigned vlen; char *val; char key[]; } brec; +typedef struct __attribute__((aligned(64))) { + volatile unsigned version; /* even = stable, odd = writer in bucket */ + volatile unsigned lock; /* writers only */ + unsigned char tags[BSLOTS]; /* 1 byte of hash per slot */ + unsigned char used; + unsigned char _pad; + brec *slot[BSLOTS]; +} bbucket; /* 4+4+6+1+1+48 = 64 */ +static bbucket *B; + +static char (*keys)[20]; +static volatile int go, stop; + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static inline void spin_lock(volatile unsigned *l){ while(__sync_lock_test_and_set(l,1)) while(*l) __builtin_ia32_pause(); } +static inline void spin_unlock(volatile unsigned *l){ __sync_lock_release(l); } +static inline void aspin_lock(volatile int *l){ while(__sync_lock_test_and_set(l,1)) while(*l) __builtin_ia32_pause(); } +static inline void aspin_unlock(volatile int *l){ __sync_lock_release(l); } + +/* ---------------- workers ---------------- */ +struct arg { int id, nthr, design, wpct; unsigned long ops; }; + +static void *worker(void *p) +{ + struct arg *a = p; + unsigned seed = 12345 + a->id * 7919; + unsigned long ops = 0; + char vbuf[VALLEN]; + + while (!go) __builtin_ia32_pause(); + + while (!stop) { + for (int rep = 0; rep < 512; rep++) { + seed = seed * 1103515245u + 12345u; + int ki = (seed >> 8) % NKEYS; + int isw = a->wpct && ((seed >> 3) % 100) < (unsigned)a->wpct; + str k = { keys[ki], KLEN }; + + if (a->design == 0) { + unsigned b = core_hash(&k, NULL, ANB); + aspin_lock(&A[b].lock); /* read takes the lock */ + for (centry *e = A[b].e; e; e = e->next) + if (e->attr.len == KLEN && memcmp(e->attr.s, k.s, KLEN) == 0) { + memcpy(vbuf, e->value.s, 8); + if (isw) e->expires++; /* TTL bump */ + break; + } + aspin_unlock(&A[b].lock); + } else { + unsigned h = core_hash(&k, NULL, 0); + unsigned b = h & (BNB - 1); + unsigned char tag = (unsigned char)(h >> 24) | 1; + + if (!isw) { /* optimistic read */ + unsigned v1, v2; + do { + v1 = __atomic_load_n(&B[b].version, __ATOMIC_ACQUIRE); + if (v1 & 1) { __builtin_ia32_pause(); continue; } + for (int s = 0; s < BSLOTS; s++) { + if (B[b].tags[s] != tag) continue; + brec *r = B[b].slot[s]; + if (!r || r->klen != KLEN) continue; + if (memcmp(r->key, k.s, KLEN) == 0) { memcpy(vbuf, r->val, 8); break; } + } + __atomic_thread_fence(__ATOMIC_ACQUIRE); + v2 = __atomic_load_n(&B[b].version, __ATOMIC_RELAXED); + } while (v1 != v2 || (v1 & 1)); + } else { /* writer */ + spin_lock(&B[b].lock); + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + for (int s = 0; s < BSLOTS; s++) + if (B[b].tags[s] == tag && B[b].slot[s] && + memcmp(B[b].slot[s]->key, k.s, KLEN) == 0) break; + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + spin_unlock(&B[b].lock); + } + } + ops++; + } + } + a->ops = ops; + return NULL; +} + +static double run(int nthr, int design, int wpct) +{ + pthread_t th[16]; struct arg ar[16]; + go = stop = 0; + for (int i = 0; i < nthr; i++) { + ar[i] = (struct arg){ i, nthr, design, wpct, 0 }; + pthread_create(&th[i], NULL, worker, &ar[i]); + } + double t0 = now(); go = 1; + struct timespec ts = { SECS, 0 }; nanosleep(&ts, NULL); + stop = 1; + unsigned long tot = 0; + for (int i = 0; i < nthr; i++) { pthread_join(th[i], NULL); tot += ar[i].ops; } + return tot / (now() - t0) / 1e6; /* Mops/sec */ +} + +int main(void) +{ + keys = malloc(NKEYS * 20); + for (int i = 0; i < NKEYS; i++) sprintf(keys[i], "%016x", (unsigned)(i * 2654435761u)); + + A = aligned_alloc(64, ANB * sizeof *A); memset(A, 0, ANB * sizeof *A); + B = aligned_alloc(64, BNB * sizeof *B); memset(B, 0, BNB * sizeof *B); + printf("sizeof(bbucket) = %zu (want 64)\n", sizeof(bbucket)); + + int placed = 0; + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], KLEN }; + unsigned h = core_hash(&k, NULL, 0); + + centry *e = malloc(sizeof(centry) + KLEN + VALLEN); + memset(e, 0, sizeof *e); + e->attr.s = (char*)e + sizeof(centry); memcpy(e->attr.s, k.s, KLEN); e->attr.len = KLEN; + e->value.s = e->attr.s + KLEN; e->value.len = VALLEN; + unsigned ab = h & (ANB - 1); + e->next = A[ab].e; A[ab].e = e; + + brec *r = malloc(sizeof(brec) + KLEN + VALLEN); + r->klen = KLEN; r->vlen = VALLEN; memcpy(r->key, k.s, KLEN); r->val = r->key + KLEN; + unsigned bb = h & (BNB - 1); + for (int s = 0; s < BSLOTS; s++) + if (!B[bb].slot[s]) { B[bb].slot[s] = r; B[bb].tags[s] = (unsigned char)(h >> 24) | 1; + B[bb].used++; placed++; break; } + } + printf("%d/%d keys placed in B (%.1f%% bucket occupancy)\n\n", + placed, NKEYS, 100.0 * placed / (BNB * BSLOTS)); + + int thr[] = { 1, 2, 4, 8 }; + for (int w = 0; w < 2; w++) { + int wpct = w ? 5 : 0; + printf("== %s ==\n", wpct ? "95% read / 5% write" : "100% read"); + printf("%-8s %12s %12s %10s\n", "threads", "A (Mops/s)", "B (Mops/s)", "B/A"); + double a1 = 0, b1 = 0; + for (unsigned i = 0; i < sizeof(thr)/sizeof(*thr); i++) { + double a = run(thr[i], 0, wpct); + double b = run(thr[i], 1, wpct); + if (!i) { a1 = a; b1 = b; } + printf("%-8d %12.2f %12.2f %9.2fx\n", thr[i], a, b, b/a); + if (thr[i] == 8) + printf("%-8s %11.2fx %11.2fx %10s <- scaling 1->8\n","scaling",a/a1,b/b1,""); + } + printf("\n"); + } + return 0; +} diff --git a/modules/cachedb_perf/bench/expire2.c b/modules/cachedb_perf/bench/expire2.c new file mode 100644 index 00000000000..fcef7a171cd --- /dev/null +++ b/modules/cachedb_perf/bench/expire2.c @@ -0,0 +1,147 @@ +/* + * Expiry strategies, corrected: + * - three independent populated tables so each strategy does REAL removals + * - realistic spread: TTLs over 3600 ticks, sweep every tick -> ~14 due per sweep + * - all accumulators volatile + printed, so nothing is optimised away + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned int core_hash(const str *s1,const str *s2,const unsigned size) +{ char *p,*end; register unsigned v; register unsigned h=0; + end=s1->s+s1->len; + for(p=s1->s;p<=(end-4);p+=4){v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3];ch_h_inc;} + v=0; for(;p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; } + +#define NKEYS 50000 +#define NBUCK 65536 +#define VALLEN 200 +#define SPREAD 3600 /* expiries spread over an hour of ticks */ +#define WHEEL 4096 /* > SPREAD, so no wrap collisions */ +#define SWEEPS 1000 +#define BASE 1000 + +static char keys[NKEYS][20]; +static int klen; +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static void *salloc(size_t s){void*p=malloc(s);(void)!malloc(16+(rand()&127));return p;} + +typedef struct ent { + str attr, value; unsigned expires, ttl; int synced; struct ent *next; + struct ent *wnext, **wpp; /* wheel links (C only) */ +} ent; +typedef struct { ent *e; volatile int lock; unsigned min_exp; } bucket; + +static volatile int sink; +static inline void lk(volatile int *l){ __sync_lock_test_and_set(l,1); sink+=*l; } +static inline void ul(volatile int *l){ __sync_lock_release(l); } + +static volatile long reap_a, reap_b, reap_c, lock_a, lock_b; + +static bucket *build(int with_min) +{ + bucket *T=calloc(NBUCK,sizeof *T); + for(int i=0;iattr.s=(char*)e+sizeof(ent); memcpy(e->attr.s,k.s,klen); e->attr.len=klen; + e->expires=BASE+(i%SPREAD); + e->next=T[b].e; T[b].e=e; + if(with_min && e->expiresexpires; + } + return T; +} + +int main(void) +{ + srand(12345); + for(int i=0;inext){ + unsigned sl=e->expires&(WHEEL-1); + e->wnext=wheel[sl]; if(wheel[sl]) wheel[sl]->wpp=&e->wnext; + wheel[sl]=e; e->wpp=&wheel[sl]; + } + double wbuild=(now()-t)*1e9/NKEYS; + printf("== hot-path cost of maintaining the index ==\n"); + printf(" wheel link on insert %8.1f ns/entry\n",wbuild); + printf(" extra memory: 2 ptr/entry = 16 B %8.1f MB @ 1M entries\n\n",16e6/1048576.0); + + /* ---------- A: full sweep, lock every bucket ---------- */ + t=now(); + for(int s=0;sexpires && e->expiresnext; free(e); reap_a++; } + else pp=&e->next; } + ul(&A[i].lock); + } + } + double ta=(now()-t)*1e3/SWEEPS; + + /* ---------- B: per-bucket min-expires hint, unlocked skip ---------- */ + t=now(); + for(int s=0;s=nowt) continue; /* plain unlocked read */ + lk(&B[i].lock); lock_b++; + unsigned mn=0xffffffffu; + ent **pp=&B[i].e; + while(*pp){ ent *e=*pp; + if(e->expires && e->expiresnext; free(e); reap_b++; } + else { if(e->expires && e->expiresexpires; pp=&e->next; } } + B[i].min_exp=mn; + ul(&B[i].lock); + } + } + double tb=(now()-t)*1e3/SWEEPS; + + /* ---------- C: timer wheel, O(expired) ---------- */ + t=now(); + for(int s=0;swnext; + /* real impl: take that entry's bucket lock, unlink from chain, free */ + reap_c++; e=nx; } + wheel[sl]=NULL; + } + double tc=(now()-t)*1e3/SWEEPS; + + printf("== cost of ONE sweep ==\n"); + printf(" A full sweep, lock every bucket %9.4f ms (%ld locks/sweep, %ld reaped)\n", + ta,lock_a/SWEEPS,reap_a); + printf(" B min-expires hint, unlocked skip %9.4f ms (%ld locks/sweep, %ld reaped)\n", + tb,lock_b/SWEEPS,reap_b); + printf(" C timer wheel, O(expired) %9.4f ms (%ld reaped)\n\n",tc,reap_c); + + printf(" B is %6.1fx cheaper than A\n",ta/tb); + printf(" C is %6.1fx cheaper than A\n\n",ta/tc); + printf(" sustained cost at a 1-second sweep interval:\n"); + printf(" A %6.3f%% of one core B %6.3f%% C %6.4f%%\n",ta/10.0,tb/10.0,tc/10.0); + return 0; +} diff --git a/modules/cachedb_perf/bench/hashtest.c b/modules/cachedb_perf/bench/hashtest.c new file mode 100644 index 00000000000..9bc01a62f4c --- /dev/null +++ b/modules/cachedb_perf/bench/hashtest.c @@ -0,0 +1,106 @@ +/* Measure core_hash() distribution vs alternatives on realistic cachedb_local keys */ +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; + +/* ---- verbatim from opensips hash_func.h ---- */ +#define ch_h_inc h+=v^(v>>3) +static inline unsigned int core_hash(const str *s1, const str *s2, const unsigned int size) +{ + char *p, *end; + register unsigned v; + register unsigned h; + h=0; + end=s1->s+s1->len; + for ( p=s1->s ; p<=(end-4) ; p+=4 ){ + v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3]; + ch_h_inc; + } + v=0; + for (; ps+s2->len; + for (p=s2->s; p<=(end-4); p+=4){ + v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3]; + ch_h_inc; + } + v=0; + for (; p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; +} + +/* ---- candidate: FNV-1a 64 + fibonacci/murmur finalizer ---- */ +static inline uint64_t fnv1a(const char *s, int len) +{ + uint64_t h = 1469598103934665603ULL; + for (int i = 0; i < len; i++) { h ^= (unsigned char)s[i]; h *= 1099511628211ULL; } + h ^= h >> 33; h *= 0xff51afd7ed558ccdULL; h ^= h >> 33; + return h; +} + +/* ---- key generators ---- */ +static void hexkey(char *b, int i) { sprintf(b, "%016x", (unsigned)(i * 2654435761u)); } /* TH thid style */ +static void dlgkey(char *b, int i) { sprintf(b, "dlg_%d_%d", i, i * 7919); } +static void aorkey(char *b, int i) { sprintf(b, "%d@sip4c.au.voipcloud.dev", 500000 + i); } +static void callidkey(char *b, int i) { sprintf(b, "%08x-%04x-4%03x@10.22.23.%d", i*2654435761u, i&0xffff, i&0xfff, 100+(i%150)); } + +struct kind { const char *name; void (*gen)(char*,int); }; + +static void run(const char *name, void (*gen)(char*,int), int n, int nbuck) +{ + int *c1 = calloc(nbuck, sizeof(int)); + int *c2 = calloc(nbuck, sizeof(int)); + char buf[128]; str s; + + for (int i = 0; i < n; i++) { + gen(buf, i); + s.s = buf; s.len = strlen(buf); + c1[core_hash(&s, NULL, nbuck)]++; + c2[fnv1a(buf, s.len) & (nbuck - 1)]++; + } + + double ideal = (double)n / nbuck; + for (int pass = 0; pass < 2; pass++) { + int *c = pass ? c2 : c1; + int empty = 0, max = 0; double chi = 0, walk = 0; + for (int i = 0; i < nbuck; i++) { + if (!c[i]) empty++; + if (c[i] > max) max = c[i]; + chi += (c[i] - ideal) * (c[i] - ideal) / ideal; + /* expected compares for a successful lookup landing in this bucket */ + walk += (double)c[i] * (c[i] + 1) / 2.0; + } + printf(" %-22s %-9s buckets=%-6d empty=%5.1f%% max_chain=%-5d chi2/df=%6.2f avg_cmp=%5.1f\n", + name, pass ? "fnv1a" : "core_hash", nbuck, + 100.0 * empty / nbuck, max, chi / nbuck, walk / n); + } + free(c1); free(c2); +} + +int main(void) +{ + struct kind kinds[] = { + {"th 16-hex thid", hexkey}, + {"dialog id", dlgkey}, + {"usrloc aor", aorkey}, + {"call-id", callidkey}, + }; + int n = 50000; + + printf("== %d keys ==\n", n); + for (int b = 0; b < 2; b++) { + int nb = b ? 65536 : 512; + printf("\n-- hash_size %s (%d buckets), load factor %.1f --\n", + b ? "16" : "9 (default)", nb, (double)n / nb); + for (unsigned k = 0; k < sizeof(kinds)/sizeof(*kinds); k++) + run(kinds[k].name, kinds[k].gen, n, nb); + } + return 0; +} diff --git a/modules/cachedb_perf/bench/hugetlb.c b/modules/cachedb_perf/bench/hugetlb.c new file mode 100644 index 00000000000..85faf4aa3ec --- /dev/null +++ b/modules/cachedb_perf/bench/hugetlb.c @@ -0,0 +1,136 @@ +/* + * Do huge pages actually help a cachedb_perf-shaped workload? + * + * Compares, over the SAME region size: + * A MAP_SHARED|MAP_ANONYMOUS <- exactly what OpenSIPS shm does + * B MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB (2 MB pages) + * + * Two access patterns, because they stress the TLB very differently: + * independent : random reads, CPU overlaps many misses (hides TLB cost) + * dependent : pointer chase, one miss at a time (exposes TLB cost) + * A hash lookup is a dependent chain: bucket -> deref slot -> compare key. + * + * Also times MADV_POPULATE_WRITE (Linux 5.14+) as a pre-fault mechanism. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0x40000 +#endif +#ifndef MADV_POPULATE_WRITE +#define MADV_POPULATE_WRITE 23 +#endif + +#define REGION (256UL*1024*1024) +#define LINE 64 +#define NLINE (REGION/LINE) +#define NITER 20000000UL + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} + +static void *try_map(int huge) +{ + int fl = MAP_SHARED|MAP_ANONYMOUS | (huge?MAP_HUGETLB:0); + void *p = mmap(NULL, REGION, PROT_READ|PROT_WRITE, fl, -1, 0); + if (p == MAP_FAILED) return NULL; + return p; +} + +/* build a random cyclic pointer chase through the region */ +static void build_chain(char *base) +{ + unsigned long n = NLINE; + unsigned long *ord = malloc(n*sizeof(unsigned long)); + for (unsigned long i=0;i0;i--){ + seed=seed*1103515245u+12345u; + unsigned long j=(unsigned long)(seed>>8)%(i+1); + unsigned long t=ord[i]; ord[i]=ord[j]; ord[j]=t; + } + for (unsigned long i=0;i>8)%NLINE)*LINE); + } + double el=now()-t; (void)sink; + return el*1e9/iters; +} + +static double depend(char *base, unsigned long iters) +{ + void *p = *(void**)base; + double t=now(); + for (unsigned long i=0;i +#include +#include +#include +#include +#include +#include +#include + +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#ifndef MAP_HUGE_SHIFT +#define MAP_HUGE_SHIFT 26 +#endif +#define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT) +#define MAP_HUGE_1GB (30 << MAP_HUGE_SHIFT) +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0x40000 +#endif + +#define REGION_DEF (256UL*1024*1024) +static unsigned long REGION = REGION_DEF; +#define LINE 64 + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} + +static long meminfo(const char *key) +{ + FILE *f = fopen("/proc/meminfo","r"); char k[64]; long v = -1; + if (!f) return -1; + while (fscanf(f, "%63s %ld kB\n", k, &v) == 2) + if (!strncmp(k, key, strlen(key))) { fclose(f); return v; } + fclose(f); return -1; +} + +static void build_chain(char *base) +{ + unsigned long n = REGION/LINE; + unsigned long *ord = malloc(n*sizeof *ord); + for (unsigned long i=0;i0;i--){ + seed=seed*1103515245u+12345u; + unsigned long j=(unsigned long)(seed>>8)%(i+1); + unsigned long t=ord[i]; ord[i]=ord[j]; ord[j]=t; + } + for (unsigned long i=0;i 1 ? argv[1] : "base"; + if (argc > 2) REGION = strtoul(argv[2], NULL, 0) * 1024UL * 1024UL; + + long shmem0 = meminfo("ShmemHugePages"), huge0 = meminfo("HugePages_Free"); + + int fl = MAP_SHARED|MAP_ANONYMOUS; + if (!strcmp(mode,"hugetlb")) fl |= MAP_HUGETLB|MAP_HUGE_2MB; + if (!strcmp(mode,"huge1g")) fl |= MAP_HUGETLB|MAP_HUGE_1GB; + + void *p = mmap(NULL, REGION, PROT_READ|PROT_WRITE, fl, -1, 0); + if (p == MAP_FAILED) { printf("%-9s mmap FAILED: %s\n", mode, strerror(errno)); return 2; } + + if (!strcmp(mode,"madvise")) + if (madvise(p, REGION, MADV_HUGEPAGE)) + printf("%-9s MADV_HUGEPAGE: %s\n", mode, strerror(errno)); + + /* fill = the fault cost, timed */ + double t = now(); + memset(p, 1, REGION); + double t_fill = now() - t; + + double t_col = 0; + if (!strcmp(mode,"collapse")) { + t = now(); + if (madvise(p, REGION, MADV_COLLAPSE)) { + printf("%-9s MADV_COLLAPSE FAILED: %s\n", mode, strerror(errno)); + munmap(p, REGION); return 3; + } + t_col = now() - t; + } + + long shmem1 = meminfo("ShmemHugePages"), huge1 = meminfo("HugePages_Free"); + long huge_mb = (shmem1-shmem0)/1024 + (huge0-huge1)*2; /* THP kB delta + hugetlb 2M pages used */ + + build_chain(p); + + /* dependent chase */ + unsigned long iters = 5000000; + void *q = *(void**)p; + t = now(); + for (unsigned long i=0;i>8)%n)*LINE); + } + double indep = (now()-t)*1e9/20000000UL; (void)sink; + + printf("%-9s fill %7.1f ms%s huge %4ld/%lu MB indep %6.2f ns chase %7.2f ns\n", + mode, t_fill*1e3, + t_col ? ({ static char b[32]; snprintf(b,sizeof b," +collapse %.0f ms",t_col*1e3); b; }) : "", + huge_mb, REGION/1024/1024, indep, chase); + + munmap(p, REGION); + return 0; +} diff --git a/modules/cachedb_perf/bench/lookup.c b/modules/cachedb_perf/bench/lookup.c new file mode 100644 index 00000000000..a2ee3ce2ea0 --- /dev/null +++ b/modules/cachedb_perf/bench/lookup.c @@ -0,0 +1,121 @@ +/* Cost of a cachedb_local lookup: chain walk + strncmp, vs stored-hash, vs resized */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; + +#define ch_h_inc h+=v^(v>>3) +static inline unsigned int core_hash(const str *s1, const str *s2, const unsigned int size) +{ + char *p, *end; register unsigned v; register unsigned h = 0; + end=s1->s+s1->len; + for ( p=s1->s ; p<=(end-4) ; p+=4 ){ v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3]; ch_h_inc; } + v=0; for (; p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; +} + +/* current entry layout, verbatim field order */ +typedef struct lcache_entry { + str attr; str value; + unsigned int expires; unsigned int ttl; int synced; + struct lcache_entry *next; +} entry_t; + +/* proposed: hash cached in the entry, no redundant pointers */ +typedef struct entry2 { + struct entry2 *next; + unsigned int hash; + unsigned int expires; + unsigned short attr_len; unsigned short pad; + unsigned int val_len; + char data[]; +} entry2_t; + +typedef struct { entry_t *e; char _pad[8]; } bucket_t; /* ptr + lock, as today */ +typedef struct { entry2_t *e; char _pad[8]; } bucket2_t; + +#define NKEYS 50000 +#define VALLEN 200 +#define ITERS 2000000 + +static char keys[NKEYS][20]; + +static double now(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+1e-9*t.tv_nsec; } + +/* scatter allocations the way shm_malloc would after a busy run */ +static void *scatter_alloc(size_t sz) +{ + void *p = malloc(sz); + void *junk = malloc(16 + (rand() & 127)); /* fragment the heap between entries */ + (void)junk; + return p; +} + +int main(void) +{ + srand(12345); + for (int i = 0; i < NKEYS; i++) sprintf(keys[i], "%016x", (unsigned)(i * 2654435761u)); + + printf("sizeof(lcache_entry_t) = %zu proposed = %zu\n\n", + sizeof(entry_t), sizeof(entry2_t)); + + int sizes[] = { 512, 65536 }; + for (int si = 0; si < 2; si++) { + int nb = sizes[si]; + bucket_t *t1 = calloc(nb, sizeof(bucket_t)); + bucket2_t *t2 = calloc(nb, sizeof(bucket2_t)); + + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], (int)strlen(keys[i]) }; + unsigned h = core_hash(&k, NULL, 0); + unsigned b = h & (nb - 1); + + entry_t *e = scatter_alloc(sizeof(entry_t) + k.len + VALLEN); + memset(e, 0, sizeof *e); + e->attr.s = (char*)e + sizeof(entry_t); memcpy(e->attr.s, k.s, k.len); e->attr.len = k.len; + e->value.s = e->attr.s + k.len; e->value.len = VALLEN; + e->next = t1[b].e; t1[b].e = e; + + entry2_t *f = scatter_alloc(sizeof(entry2_t) + k.len + VALLEN); + f->hash = h; f->attr_len = k.len; f->val_len = VALLEN; f->expires = 0; + memcpy(f->data, k.s, k.len); + f->next = t2[b].e; t2[b].e = f; + } + + /* --- A: current --- */ + double t = now(); unsigned long hits = 0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i * 7919) % NKEYS); + str k = { keys[ki], (int)strlen(keys[ki]) }; + unsigned b = core_hash(&k, NULL, nb); + for (entry_t *e = t1[b].e; e; e = e->next) + if (e->attr.len == k.len && strncmp(e->attr.s, k.s, k.len) == 0) { hits++; break; } + } + double ta = now() - t; + + /* --- B: hash stored in entry, memcmp only on hash match --- */ + t = now(); unsigned long hits2 = 0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i * 7919) % NKEYS); + str k = { keys[ki], (int)strlen(keys[ki]) }; + unsigned h = core_hash(&k, NULL, 0); + unsigned b = h & (nb - 1); + for (entry2_t *e = t2[b].e; e; e = e->next) + if (e->hash == h && e->attr_len == k.len && + memcmp(e->data, k.s, k.len) == 0) { hits2++; break; } + } + double tb = now() - t; + + printf("-- %d buckets (load %.1f) --\n", nb, (double)NKEYS/nb); + printf(" A current (walk + strncmp) : %6.2f s %8.1f ns/lookup (%lu hits)\n", + ta, ta*1e9/ITERS, hits); + printf(" B stored hash + memcmp : %6.2f s %8.1f ns/lookup (%.2fx)\n", + tb, tb*1e9/ITERS, ta/tb); + } + return 0; +} diff --git a/modules/cachedb_perf/bench/mlockt.c b/modules/cachedb_perf/bench/mlockt.c new file mode 100644 index 00000000000..de07cb6f131 --- /dev/null +++ b/modules/cachedb_perf/bench/mlockt.c @@ -0,0 +1,94 @@ +/* + * Can the arena be pinned against swap, and what does it cost? + * + * - mlock() on the MAP_SHARED|MAP_ANON region OpenSIPS shm uses + * - verified via /proc/meminfo Mlocked/Unevictable deltas, never assumed + * - fork test: locks are NOT inherited (man mlock2), but the pages are + * SHARED - so a lock held by the pre-fork process pins them for every + * worker. This decides where the call must live: mod_init. + * - mlock doubles as a pre-fault (it must populate to pin) - timed against + * MADV_POPULATE_WRITE. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef MADV_POPULATE_WRITE +#define MADV_POPULATE_WRITE 23 +#endif + +static unsigned long REGION = 256UL*1024*1024; + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static long mi(const char *key) +{ + FILE *f = fopen("/proc/meminfo","r"); char k[64]; long v=-1; + while (f && fscanf(f,"%63s %ld kB\n",k,&v)==2) + if (!strncmp(k,key,strlen(key))) { fclose(f); return v; } + if (f) fclose(f); return -1; +} + +int main(int argc, char **argv) +{ + if (argc > 1) REGION = strtoul(argv[1],NULL,0)*1024UL*1024UL; + + struct rlimit rl; + getrlimit(RLIMIT_MEMLOCK, &rl); + printf("RLIMIT_MEMLOCK: cur=%ld KB (region: %lu MB)\n", + rl.rlim_cur==RLIM_INFINITY?-1:(long)(rl.rlim_cur/1024), REGION/1024/1024); + + void *p = mmap(NULL, REGION, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS, -1, 0); + if (p==MAP_FAILED){perror("mmap");return 1;} + + long l0 = mi("Mlocked"), u0 = mi("Unevictable"); + + /* cold mlock: populates AND pins in one call */ + double t = now(); + if (mlock(p, REGION)) { printf("mlock FAILED: %s\n", strerror(errno)); return 2; } + double t_mlock = now()-t; + + long l1 = mi("Mlocked"), u1 = mi("Unevictable"); + printf("mlock cold (populate+pin): %7.1f ms Mlocked +%ld MB, Unevictable +%ld MB\n", + t_mlock*1e3, (l1-l0)/1024, (u1-u0)/1024); + + /* fork: child has NO lock of its own, but pages stay pinned because the + * parent (i.e. the pre-fork attendant in OpenSIPS) still holds the lock */ + pid_t pid = fork(); + if (pid == 0) { + memset(p, 7, REGION); /* child writes shared pages */ + long lc = mi("Mlocked"); + printf("in child (no own lock): Mlocked still +%ld MB globally\n", + (lc-l0)/1024); + fflush(stdout); + _exit(0); + } + waitpid(pid, NULL, 0); + + munlock(p, REGION); + long l2 = mi("Mlocked"); + printf("after munlock: Mlocked +%ld MB (back to baseline)\n", + (l2-l0)/1024); + + /* compare: populate-then-lock as two steps */ + munmap(p, REGION); + p = mmap(NULL, REGION, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + t = now(); + int rc = madvise(p, REGION, MADV_POPULATE_WRITE); + double t_pop = now()-t; + t = now(); + rc |= mlock(p, REGION); + double t_lock2 = now()-t; + if (!rc) + printf("two-step: POPULATE_WRITE %7.1f ms + warm mlock %7.1f ms = %7.1f ms total\n", + t_pop*1e3, t_lock2*1e3, (t_pop+t_lock2)*1e3); + munlock(p, REGION); munmap(p, REGION); + return 0; +} diff --git a/modules/cachedb_perf/bench/pullsoak/E2E.md b/modules/cachedb_perf/bench/pullsoak/E2E.md new file mode 100644 index 00000000000..3d079206680 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/E2E.md @@ -0,0 +1,49 @@ +# Blocking versus suspending, end to end + +Two nodes, four workers each, and a burst of requests to the node that +does **not** have the state - so every one of them costs a round trip to +the other node. The same workload runs two ways: + +```bash +MODE=sync sh e2e_rig.sh && python3 e2e_burst.py sync +MODE=async sh e2e_rig.sh && python3 e2e_burst.py async +``` + +`sync` sets `pull_on_miss` and calls `topology_hiding_match()`, so the +lookup blocks its worker until the cluster answers. `async` clears +`pull_on_miss` and calls `async(topology_hiding_match(), resume)`, so the +transaction suspends and the worker goes back to work. Both must be +configured, or the comparison is meaningless: with `pull_on_miss` set, +the ordinary lookup pulls before the asynchronous path ever sees a miss. + +## What it shows + +On an idle network, **nothing**: 120 requests took 0.044 s blocking and +0.046 s suspending. A pull takes well under a millisecond there, and +holding a worker for that long costs nothing measurable. This is worth +knowing - it is why `pull_on_miss` is not dangerous on a quiet LAN, and +why the asynchronous path is not a general speed-up. + +Add 100 ms to the peer's replies (`tc qdisc add dev y1 root netem delay +100ms` inside the peer's namespace) and the difference is the whole +point: + +| | 60 requests, 4 workers, peer 100 ms away | +|---|---| +| blocking | 1.52 s | +| suspending | 0.31 s | + +The blocking figure is arithmetic, not noise: sixty round trips of a +tenth of a second, four at a time, is a second and a half. The other +path finishes in about one round trip however many requests there are, +because none of them is holding anything while it waits. + +## Traps + +The key must be exactly `TH_KEY_LEN` characters or the store rejects it +by length, which looks exactly like a lookup that found nothing - an +earlier version of this test generated keys of 15, 16 and 17 characters +and "failed" only for the ones that were not 16. + +Clear both caches before each run. A pull stores what it fetched, so a +second run measures local hits and reports a number that means nothing. diff --git a/modules/cachedb_perf/bench/pullsoak/README.md b/modules/cachedb_perf/bench/pullsoak/README.md new file mode 100644 index 00000000000..69b450f6bcb --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/README.md @@ -0,0 +1,49 @@ +# Cross-node pull soak + +Unlike the rest of `bench/`, this is **not a model** — it drives two real +OpenSIPS instances with the real module, over the real clusterer transport, +and asserts on behaviour rather than measuring a number. + +```bash +sh rig.sh # two nodes on loopback, 8 workers each, pull enabled +python3 soak.py # 12 concurrent threads; prints PASS/FAIL per invariant +``` + +## Why it exists + +Three defects in the pull path were found by *reading* the code, after the +sequential tests had passed 32/32 both before and after the fix: + +| defect | why the sequential tests were blind to it | +|---|---| +| the pull lock was held across the table write | no two pulls ever overlapped, so nothing contended | +| negative replies were not deduped by sender | one peer, one reply — a second could not arrive | +| the pull counters were non-atomic | one writer at a time loses nothing | + +All three need *concurrency* to show themselves. This soak supplies it: 8 +threads pulling, 2 asking for keys that exist nowhere, and 2 rewriting the +same keys the others are pulling — so slots are contended, replies +interleave, and writes race reads on the same buckets. + +## What each assertion is actually watching for + +| assertion | the regression it catches | +|---|---| +| no thread hung or errored | a lock-ordering deadlock, or a stall from holding a lock across I/O | +| no reply matched to the wrong request | correlation broken — the value carries its own key's index, so a cross-matched reply is a number that does not belong | +| no round came back empty | a false cluster-wide "absent" for keys that demonstrably exist — what a missing per-sender dedupe produces | +| keys that exist nowhere are never invented | the negative path answering positively | +| `stored <= received <= requested` | lost counter updates | +| `pulls_in_flight` back to 0 | slots taken and never released, which ends as silent loss of read repair | +| no crash, no slot exhaustion | the obvious ones | + +## Notes + +The stored value is the key's own index, and the check is numeric. That is +not a stylistic choice: a quoted string is expanded only in a *function +argument*, never in an assignment or a comparison, so there is no way to +build an expected string in the script and compare against it. Two earlier +attempts to do so reported every value as wrong. + +Header values arrive as strings — the loop bounds are cast with `{s.int}` or +the counter never increments and the loop runs to `max_while_loops`. diff --git a/modules/cachedb_perf/bench/pullsoak/async_leak_test.py b/modules/cachedb_perf/bench/pullsoak/async_leak_test.py new file mode 100644 index 00000000000..bb2fe706600 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/async_leak_test.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""The ASYNC path must not hold a slot when no conclusive answer arrives. + +This is the path the leak lives on. A first attempt drove `perf_pull` (MI) +instead, and it passed on the UNFIXED build - because MI uses the blocking +entry point, which polls for pull_timeout_ms and then calls finish() +regardless, so it can never leak. Only async(topology_hiding_match()) +suspends on the eventfd and depends on the reply handler to arm it. + +Reproduction: seed the state on n1 only, freeze n1 with SIGSTOP so the +request still goes out but no answer can come back, then drive the async +match on n2. `negative` never reaches `expect`, the fd is never armed, and +before the reaper the slot was held for ever. + +Run on /dn/thasync (netns rig). +""" +import json, os, signal, socket, subprocess, sys, time + +D = "/dn/thasync" +KEY = "a1b2c3d4e5f60718" + +def mi(n, method, params=None, timeout=20): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("/tmp/al.%d.%f" % (os.getpid(), time.time())); c.settimeout(timeout) + c.sendto(json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, + "params": params if params is not None else []}).encode(), + "%s/mi%d.sock" % (D, n)) + try: + r = json.loads(c.recv(262144)) + except socket.timeout: + return None + finally: + c.close() + return r.get("result", r.get("error")) + +def sip(node, hdr, ruri_param="", timeout=60): + ip = "10.97.0.%d" % node + prog = ("import socket,time\n" + "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM)\n" + "s.bind(('%s',0)); p=s.getsockname()[1]\n" + "m=('BYE sip:uas@%s:5060%s SIP/2.0\\r\\n'\n" + " 'Via: SIP/2.0/UDP %s:%%d;branch=z9hG4bK-%%f\\r\\n'\n" + " 'From: ;tag=aa\\r\\nTo: ;tag=bb\\r\\n'\n" + " 'Call-ID: leak-%%f\\r\\nCSeq: 2 BYE\\r\\n%s'\n" + " 'Max-Forwards: 70\\r\\nContent-Length: 0\\r\\n\\r\\n')" + "%%(p,time.time(),time.time())\n" + "s.settimeout(%d); s.sendto(m.encode(),('%s',5060))\n" + "try: print(s.recvfrom(65535)[0].decode().split('\\r\\n')[0])\n" + "except Exception as e: print('NO-REPLY', e)\n" + ) % (ip, ip, ruri_param, ip, hdr.replace("\r\n", "\\r\\n"), + timeout, ip) + out = subprocess.run(["ip", "netns", "exec", "t%d" % node, + "python3", "-c", prog], + capture_output=True, text=True, timeout=timeout + 30) + return (out.stdout or out.stderr).strip() + +def cl(n): + return (mi(n, "perf_stats") or {}).get("cluster", {}) + +def pids(cfg): + out = subprocess.run(["pgrep", "-f", "thasync/%s" % cfg], + capture_output=True, text=True).stdout.split() + return [int(p) for p in out] + +ok = fail = 0 +def check(name, cond, detail=""): + global ok, fail + print(" %-54s %s %s" % (name, "PASS" if cond else "FAIL", detail)) + ok, fail = ok + (1 if cond else 0), fail + (0 if cond else 1) + +contact = "sip:uas@10.97.0.9:5099" +sock = "udp:10.97.0.1:5060" +blob = "0:" + "%d:%s" % (len(contact), contact) + "3:100" + \ + "%d:%s" % (len(sock), sock) + +# convergence from an earlier run would leave n2 holding it - clear both +mi(1, "perf_del", {"glob": "th:*", "collection": "th"}) +mi(2, "perf_del", {"glob": "th:*", "collection": "th"}) +mi(1, "perf_set", {"key": "th:" + KEY, "value": blob, "ttl": 300, + "collection": "th"}) +check("state seeded on n1 only", + (mi(2, "perf_probe", {"key": "th:" + KEY, "collection": "th"}) or {}) + .get("code") == 404) + +b = cl(2) +print(" before: in_flight=%s timed_out=%s abandoned=%s" + % (b.get("pulls_in_flight"), b.get("pulls_timed_out"), + b.get("pulls_abandoned"))) + +frozen = pids("n1.cfg") +check("n1 processes located to freeze", len(frozen) > 0, len(frozen)) +for p in frozen: + os.kill(p, signal.SIGSTOP) +try: + time.sleep(0.5) + r = sip(2, "X-Async: 1\r\n", ";tk=_" + KEY, timeout=20) + print(" n2 async match with n1 frozen ->", r) + time.sleep(10) # deadline + abandon grace + reaper ticks + a = cl(2) + print(" after : in_flight=%s timed_out=%s abandoned=%s" + % (a.get("pulls_in_flight"), a.get("pulls_timed_out"), + a.get("pulls_abandoned"))) + check("THE FIX: slot reclaimed (in_flight back to 0)", + a.get("pulls_in_flight") == 0, a.get("pulls_in_flight")) + check("and the pull was accounted for", + a.get("pulls_timed_out", 0) > b.get("pulls_timed_out", 0) + or a.get("pulls_abandoned", 0) > b.get("pulls_abandoned", 0)) +finally: + for p in frozen: + try: + os.kill(p, signal.SIGCONT) + except ProcessLookupError: + pass + time.sleep(5) + +check("cluster topology is reported", isinstance(cl(2).get("topology"), list) + and cl(2).get("topology"), json.dumps(cl(2).get("topology"))[:100]) + +log = open("%s/n2.log" % D, errors="replace").read() +check("n2 did not crash", + "sig_usr: segfault" not in log and "*** stack smashing" not in log) + +print("\n%d passed, %d failed" % (ok, fail)) +raise SystemExit(1 if fail else 0) diff --git a/modules/cachedb_perf/bench/pullsoak/e2e_burst.py b/modules/cachedb_perf/bench/pullsoak/e2e_burst.py new file mode 100755 index 00000000000..131badb16ae --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/e2e_burst.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""CP-15.14: a burst of cross-node misses, blocking versus suspending. + +Every request asks node 2 for a state only node 1 has, so every one of +them costs a cluster round trip. With the lookup blocking, a worker is +occupied for the whole of it and the node can have only as many in flight +as it has workers. Suspending should let far more overlap. +""" +import os, socket, subprocess, sys, threading, time + +D = "/dn/e2e" +N = int(os.environ.get("N", "120")) +CONC = int(os.environ.get("CONC", "20")) + +def sip_batch(node, keys): + """fire len(keys) requests from inside the namespace, concurrently, + and report how long the whole batch took plus each reply code""" + ip = "10.95.0.%d" % node + prog = """ +import socket, sys, threading, time +keys = sys.argv[1].split(",") +res = [] +lock = threading.Lock() +def one(k): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.bind(("%s", 0)); p = s.getsockname()[1] + m = ("OPTIONS sip:uas@%s:5060;tk=_" + k + " SIP/2.0\\r\\n" + "Via: SIP/2.0/UDP %s:" + str(p) + ";branch=z9hG4bK-" + k + "\\r\\n" + "From: ;tag=f" + k + "\\r\\nTo: ;tag=t" + k + "\\r\\n" + "Call-ID: c-" + k + "\\r\\nCSeq: 4 OPTIONS\\r\\nX-Match: 1\\r\\n" + "Max-Forwards: 70\\r\\nContent-Length: 0\\r\\n\\r\\n") + s.settimeout(15) + try: + s.sendto(m.encode(), ("%s", 5060)) + d, _ = s.recvfrom(65535) + code = d.decode(errors="replace").split()[1] + except Exception: + code = "timeout" + finally: + s.close() + with lock: + res.append(code) +t0 = time.time() +ths = [threading.Thread(target=one, args=(k,)) for k in keys] +for t in ths: t.start() +for t in ths: t.join() +print("%%.3f" %% (time.time() - t0)) +print(",".join(res)) +""" % (ip, ip, ip, ip) + out = subprocess.run(["ip", "netns", "exec", "e%d" % node, "python3", "-c", + prog, ",".join(keys)], + capture_output=True, text=True, timeout=300) + lines = (out.stdout or "").strip().split("\n") + if len(lines) < 2: + return None, (out.stderr or "")[:200] + return float(lines[0]), lines[1].split(",") + +import json +def mi(n, method, params=None): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("/tmp/e2.%d.%f" % (os.getpid(), time.time())); c.settimeout(30) + c.sendto(json.dumps({"jsonrpc":"2.0","id":1,"method":method, + "params": params if params is not None else []}).encode(), + "%s/mi%d.sock" % (D, n)) + r = json.loads(c.recv(262144)); c.close() + return r.get("result", r.get("error")) + +BLOB = "0:22:sip:uas@10.95.0.9:50993:10018:udp:10.95.0.1:5060" + +def seed_mi(keys): + """the key must be exactly TH_KEY_LEN or th_store rejects it by length + - a mistake that silently looks like a failed lookup""" + for k in keys: + assert len(k) == 16, k + mi(1, "perf_set", {"key": "th:" + k, "value": BLOB, "ttl": 600, + "collection": "th"}) + +def pulls(n): + return {k: v for k, v in mi(n, "perf_stats").get("cluster", {}).items() + if k.startswith("pull")} + +def old_seed(): + ip = "10.95.0.1" + prog = ("import socket,time\n" + "s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM); s.bind(('%s',0))\n" + "p=s.getsockname()[1]\n" + "m=('OPTIONS sip:b@%s SIP/2.0\\r\\nVia: SIP/2.0/UDP %s:'+str(p)+';branch=z9hG4bK-s\\r\\n'\n" + " 'From: ;tag=b\\r\\nTo: \\r\\nCall-ID: seed\\r\\nCSeq: 1 OPTIONS\\r\\n'\n" + " 'X-Seed: 1\\r\\nMax-Forwards: 70\\r\\nContent-Length: 0\\r\\n\\r\\n')\n" + "s.settimeout(60); s.sendto(m.encode(),('%s',5060))\n" + "print(s.recvfrom(65535)[0].decode().split()[1])\n") % (ip, ip, ip, ip) + out = subprocess.run(["ip", "netns", "exec", "e1", "python3", "-c", prog], + capture_output=True, text=True, timeout=120) + return (out.stdout or out.stderr).strip() + +mode = sys.argv[1] if len(sys.argv) > 1 else "?" +keys = ["e2ekey%010d" % i for i in range(N)] # exactly 16 characters +# clear both nodes first: a pull converges, so a previous run would leave +# n2 holding everything and the burst would measure local hits +mi(1, "perf_del", {"glob": "th:*", "collection": "th"}) +mi(2, "perf_del", {"glob": "th:*", "collection": "th"}) +seed_mi(keys) +print("seeded %d keys on n1; n2 holds %s" + % (len(keys), mi(2, "perf_stats")["collections"][0]["entries"])) +before = pulls(2) +dt, codes = sip_batch(2, keys) +after = pulls(2) +if dt is None: + print("FAILED:", codes) + sys.exit(1) +good = sum(1 for c in codes if c == "200") +print("%-6s %d requests, %d concurrent-ish: %.2fs matched=%d other=%s" + % (mode, N, CONC, dt, good, + sorted(set(c for c in codes if c != "200")) or "none")) +print(" pulls:", {k: after.get(k,0) - before.get(k,0) for k in after + if k in ("pulls_requested","pulls_received","pulls_timed_out")}) +print("RESULT %s %.3f %d" % (mode, dt, good)) diff --git a/modules/cachedb_perf/bench/pullsoak/e2e_rig.sh b/modules/cachedb_perf/bench/pullsoak/e2e_rig.sh new file mode 100755 index 00000000000..79c8f12b3eb --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/e2e_rig.sh @@ -0,0 +1,115 @@ +#!/bin/sh +# CP-15.14 async e2e: a burst of cross-node misses, handled two ways. +# MODE=sync pull_on_miss=1, script calls topology_hiding_match() +# -> the lookup blocks the worker while the cluster answers +# MODE=async pull_on_miss=0, script calls async(topology_hiding_match()) +# -> the transaction suspends, the worker goes back to work +# Same workload, same node count, same worker count. +set -e +T=/dn/wt-cp15 +D=/dn/e2e +MODE=${MODE:-async} +WORKERS=${WORKERS:-4} +mkdir -p $D +pkill -f "e2e/n[12].cfg" 2>/dev/null || true +sleep 1 +for i in 1 2; do ip netns del e$i 2>/dev/null || true; ip link del y${i}p 2>/dev/null || true; done +ip link del br95 2>/dev/null || true +sleep 1 +rm -f $D/mi1.sock $D/mi2.sock + +if [ "$MODE" = "sync" ]; then ONMISS=1; else ONMISS=0; fi + +ip link add br95 type bridge +ip link set br95 up +echo 0 > /sys/class/net/br95/bridge/multicast_snooping 2>/dev/null || true +for i in 1 2; do + ip netns add e$i + ip link add y$i type veth peer name y${i}p + ip link set y$i netns e$i + ip link set y${i}p master br95 up + ip netns exec e$i ip link set lo up + ip netns exec e$i ip addr add 10.95.0.$i/24 dev y$i + ip netns exec e$i ip link set y$i up + ip netns exec e$i ip route add 239.0.0.0/8 dev y$i + + cat > $D/n$i.cfg <> $D/n$i.cfg <<'EOF' + if (topology_hiding_match()) + sl_send_reply(200, "m"); + else + sl_send_reply(404, "n"); + exit; + } + sl_send_reply(200, "ok"); + exit; +} +EOF + else + cat >> $D/n$i.cfg <<'EOF' + async(topology_hiding_match(), r); + exit; + } + sl_send_reply(200, "ok"); + exit; +} +route[r] { + if ($rc > 0) sl_send_reply(200, "m"); else sl_send_reply(404, "n"); + exit; +} +EOF + fi + ip netns exec e$i $T/opensips -f $D/n$i.cfg -F > $D/n$i.log 2>&1 & +done +sleep 14 +grep -ahc "cross-node pull active" $D/n1.log $D/n2.log diff --git a/modules/cachedb_perf/bench/pullsoak/maxkey_test.py b/modules/cachedb_perf/bench/pullsoak/maxkey_test.py new file mode 100644 index 00000000000..3907b02f1ed --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/maxkey_test.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""The serve path now rejects out-of-range keys before it echoes them into a +fixed reply buffer. That gate sits right next to the largest key a +legitimate requester can send, so this checks the boundary from both sides: +a key of exactly PCACHE_PULL_MAX_KEY must still pull cleanly, and the value +that rides back with it must still fit the controller plane's datagram. + +Run against the live clctr netns rig (/tmp/clctrig.sh).""" +import json, os, socket, time + +D = "/dn/clctrpull" +MAXKEY = 256 # PCACHE_PULL_MAX_KEY +RPL_HDR = 14 # PCACHE_CLCTR_RPL_HDR +CLCTR_MAX = 1300 # CLCTR_MAX_PAYLOAD + +def mi(n, method, params=None): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("/tmp/mk.%d.%f" % (os.getpid(), time.time())); c.settimeout(15) + c.sendto(json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, + "params": params if params is not None else []}).encode(), + "%s/mi%d.sock" % (D, n)) + r = json.loads(c.recv(262144)); c.close() + return r.get("result", r.get("error")) + +ok = fail = 0 +def check(name, cond, detail=""): + global ok, fail + print(" %-56s %s %s" % (name, "PASS" if cond else "FAIL", detail)) + ok, fail = ok + (1 if cond else 0), fail + (0 if cond else 1) + +# a key of exactly the maximum a requester may send +key = "th:" + "k" * (MAXKEY - 3) +assert len(key) == MAXKEY +# the largest value that still fits beside it on the controller plane +val = "v" * (CLCTR_MAX - RPL_HDR - MAXKEY) +print("key %d bytes, value %d bytes, reply %d of %d\n" + % (len(key), len(val), RPL_HDR + len(key) + len(val), CLCTR_MAX)) + +mi(1, "perf_set", {"key": key, "value": val, "ttl": 600, "collection": "sync"}) +check("n1 stored the max-length key", + (mi(1, "perf_get", {"key": key, "collection": "sync"}) or {}).get("value") == val) + +# pull counters live in the cluster object; collections are keyed by "name" +b = mi(2, "perf_stats")["cluster"] + +got = mi(2, "perf_pull", {"key": key, "collection": "sync"}) +check("n2 pulled a max-length key across the cluster", + isinstance(got, dict) and got.get("value") == val, + "" if isinstance(got, dict) and got.get("value") == val else got) + +a = mi(2, "perf_stats")["cluster"] +d = {k: a[k] - b[k] for k in ("pulls_requested", "pulls_received", + "pulls_stored", "pulls_timed_out")} +print(" deltas:", d) +check("the pull was answered, not timed out", + d["pulls_received"] == 1 and d["pulls_timed_out"] == 0, d) +check("and stored locally", d["pulls_stored"] == 1, d) +check("n2 now serves it without asking again", + (mi(2, "perf_get", {"key": key, "collection": "sync"}) or {}).get("value") == val) + +for n in (1, 2): + log = open("%s/n%d.log" % (D, n)).read() + check("n%d: no key-range rejection for a legitimate key" % n, + "out of range" not in log) + check("n%d: no crash" % n, + "SIGSEGV" not in log and "CRITICAL:" not in log) + +print("\n%d passed, %d failed" % (ok, fail)) +raise SystemExit(1 if fail else 0) diff --git a/modules/cachedb_perf/bench/pullsoak/oversize_test.sh b/modules/cachedb_perf/bench/pullsoak/oversize_test.sh new file mode 100644 index 00000000000..b36750abc09 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/oversize_test.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Drive an out-of-spec key at the serve path, which no well-behaved node can +# do - so the sender's own gate is relaxed for this build only (test_relax.py). +# +# n1 = clctr transport (the node under test: it frames replies into a fixed +# 1300-byte datagram buffer) +# n2 = bin transport (the sender: BIN puts no bound on the key it can push) +# +# The mixed transport is the point: both receivers are live on n1, so a BIN +# request reaches its serve path and, before the fix, was answered by the +# clctr writer - copying an unbounded key into that fixed buffer. +set -e +D=/dn/clctrpull +KEYLEN=${KEYLEN:-5000} + +sed -i 's/pull_transport", "clctr"/pull_transport", "bin"/' $D/n2.cfg +pkill -f "clctrpull/n2.cfg" 2>/dev/null || true +sleep 1 +ip netns exec n2 /dn/wt-cp15/opensips -f $D/n2.cfg -F >> $D/n2.log 2>&1 & +sleep 8 + +python3 - "$KEYLEN" <<'PY' +import json, os, socket, sys, time +D = "/dn/clctrpull" +klen = int(sys.argv[1]) + +def mi(n, method, params=None, timeout=20): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("/tmp/ov.%d.%f" % (os.getpid(), time.time())); c.settimeout(timeout) + c.sendto(json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, + "params": params if params is not None else []}).encode(), + "%s/mi%d.sock" % (D, n)) + try: + r = json.loads(c.recv(262144)) + except socket.timeout: + return None + finally: + c.close() + return r.get("result", r.get("error")) + +n1_before = mi(1, "perf_stats") is not None +print(" n1 alive before: %s" % n1_before) + +key = "th:" + "K" * (klen - 3) +print(" n2 pulling a %d byte key over BIN, to be served by n1 (clctr)" % len(key)) +r = mi(2, "perf_pull", {"key": key, "collection": "sync"}, timeout=25) +print(" n2 perf_pull ->", str(r)[:120]) + +time.sleep(2) +alive = mi(1, "perf_stats") +print(" n1 alive after : %s" % (alive is not None)) + +log = open("%s/n1.log" % D, errors="replace").read() +crashed = ("sig_usr: segfault" in log or "core dumped" in log + or "*** stack smashing" in log) +rejected = "out of range" in log +print(" n1 logged an out-of-range rejection: %s" % rejected) +print(" n1 shows a crash/smash : %s" % crashed) +print("RESULT alive=%s rejected=%s crashed=%s" % (alive is not None, rejected, crashed)) +PY diff --git a/modules/cachedb_perf/bench/pullsoak/rig.sh b/modules/cachedb_perf/bench/pullsoak/rig.sh new file mode 100755 index 00000000000..e8982f56412 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/rig.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# Concurrent pull soak: the shape of test that would have caught the three +# defects found by reading the code (lock scope, per-sender dedupe, atomic +# counters). Several workers on BOTH nodes pull at once, while writes race +# the pulls, so slots are contended and replies interleave. +D=${D:-/tmp/cachedb_perf_soak} +T=${T:-$(cd "$(dirname "$0")/../../../.." && pwd)} +KEYS=${KEYS:-500} +mkdir -p $D +pkill -f "soak/n[12].cfg" 2>/dev/null; sleep 1 +rm -f $D/mi1.sock $D/mi2.sock $D/shared.db +sqlite3 $D/shared.db "CREATE TABLE cachedb_perf (collection TEXT, pkey TEXT, pvalue BLOB, expires INTEGER);" + +for i in 1 2; do +cat > $D/n$i.cfg < $D/n1.log 2>&1 & +./opensips -f $D/n2.cfg -F > $D/n2.log 2>&1 & +sleep 7 +grep -ahc "cross-node pull active" $D/n1.log $D/n2.log diff --git a/modules/cachedb_perf/bench/pullsoak/soak.py b/modules/cachedb_perf/bench/pullsoak/soak.py new file mode 100755 index 00000000000..41db2f8dff8 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/soak.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Concurrent pull soak - the test shape that WOULD have caught the three +defects found by reading the code back: + + lock scope -> concurrent pulls + writes deadlock or stall + sender dedupe -> a false cluster-wide "absent" while a key demonstrably + exists (ghost band vs real band) + atomic counters-> requested/received/stored stop adding up + +plus the leak the gauge now exposes: slots taken and never released. +""" +import json, os, socket, sys, threading, time + +D = os.environ.get("SOAK_DIR", "/tmp/cachedb_perf_soak") +KEYS = int(os.environ.get("KEYS", "500")) +THREADS = int(os.environ.get("THREADS", "8")) +ROUNDS = int(os.environ.get("ROUNDS", "6")) + +def mi(n, method, params=None): + c = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + c.bind("/tmp/sk.%d.%f" % (os.getpid(), time.time())); c.settimeout(20) + c.sendto(json.dumps({"jsonrpc":"2.0","id":1,"method":method, + "params": params if params is not None else []}).encode(), + "%s/mi%d.sock" % (D, n)) + r = json.loads(c.recv(262144)); c.close() + return r.get("result", r.get("error")) + +def sip(port, hdrs, timeout=60): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.bind(("127.0.0.1", 0)) + p = s.getsockname()[1] + m = ("OPTIONS sip:b@127.0.0.1 SIP/2.0\r\n" + "Via: SIP/2.0/UDP 127.0.0.1:%d;branch=z9hG4bK-%f\r\n" + "From: ;tag=b\r\nTo: \r\n" + "Call-ID: soak-%f-%d\r\nCSeq: 1 OPTIONS\r\n%s" + "Max-Forwards: 70\r\nContent-Length: 0\r\n\r\n") % ( + p, time.time(), time.time(), p, hdrs) + s.settimeout(timeout) + s.sendto(m.encode(), ("127.0.0.1", port)) + try: + d, _ = s.recvfrom(65535) + return d.decode(errors="replace").split("\r\n")[0] + finally: + s.close() + +results = [] +lock = threading.Lock() + +def hammer(tid): + try: + lo = tid * (KEYS // THREADS) + hi = lo + (KEYS // THREADS) + for _ in range(ROUNDS): + r = sip(5082, "X-Hammer: 1\r\nX-From: %d\r\nX-To: %d\r\n" % (lo, hi)) + with lock: results.append(("hammer", r)) + except Exception as e: + with lock: results.append(("hammer-EXC", str(e))) + +def ghost(tid): + try: + for _ in range(ROUNDS): + r = sip(5082, "X-Ghost: 1\r\n") + with lock: results.append(("ghost", r)) + except Exception as e: + with lock: results.append(("ghost-EXC", str(e))) + +def churn(tid): + try: + for _ in range(ROUNDS): + r = sip(5081, "X-Churn: 1\r\n") + with lock: results.append(("churn", r)) + except Exception as e: + with lock: results.append(("churn-EXC", str(e))) + +ok = fail = 0 +def check(name, cond, detail=""): + global ok, fail + print(" %-50s %s %s" % (name, "PASS" if cond else "FAIL", detail)) + ok, fail = ok + (1 if cond else 0), fail + (0 if cond else 1) + +print("seeding %d keys on n1..." % KEYS) +print(" ", sip(5081, "X-Seed: 1\r\n", timeout=120)) + +before = {k: v for k, v in mi(2, "perf_stats").get("cluster", {}).items() + if k.startswith("pull")} +print("\nrunning %d hammer + 2 ghost + 2 churn threads, %d rounds each..." + % (THREADS, ROUNDS)) +t0 = time.time() +threads = ([threading.Thread(target=hammer, args=(i,)) for i in range(THREADS)] + + [threading.Thread(target=ghost, args=(i,)) for i in range(2)] + + [threading.Thread(target=churn, args=(i,)) for i in range(2)]) +for t in threads: t.start() +for t in threads: t.join() +dt = time.time() - t0 +print(" finished in %.1fs" % dt) + +after = {k: v for k, v in mi(2, "perf_stats").get("cluster", {}).items() + if k.startswith("pull")} + +excs = [r for r in results if "EXC" in r[0]] +check("no thread hung or errored", not excs, excs[:2]) + +bad = [r for r in results if r[0] == "hammer" and "bad=0" not in r[1]] +check("no reply matched to the wrong request", not bad, bad[:2]) + +hammered = [r for r in results if r[0] == "hammer"] +check("every hammer round answered", + len(hammered) == THREADS * ROUNDS, len(hammered)) + +gh = [r for r in results if r[0] == "ghost" and "ghostfound=0" not in r[1]] +check("keys that exist nowhere are never invented", not gh, gh[:2]) + +# a real key must never be declared absent: hit counts must be full +zero = [r for r in hammered if "hit=0 " in r[1]] +check("no hammer round came back empty (no false absence)", not zero, zero[:2]) + +d = {k: after.get(k, 0) - before.get(k, 0) for k in after} +print(" deltas:", d) +check("counters add up: stored <= received <= requested", + d["pulls_stored"] <= d["pulls_received"] <= d["pulls_requested"], d) +check("nothing timed out", d["pulls_timed_out"] == 0, d["pulls_timed_out"]) + +st = mi(2, "perf_stats").get("cluster", {}) +check("all pull slots released (no leak)", + st.get("pulls_in_flight", -1) == 0, + "%s/%s" % (st.get("pulls_in_flight"), st.get("pull_slots"))) + +for n in (1, 2): + log = open("%s/n%d.log" % (D, n), errors="replace").read() + bad = [l for l in log.splitlines() + if "CRITICAL" in l or "SIGSEGV" in l or "slots busy" in l] + check("n%d: no crash or slot exhaustion" % n, not bad, bad[:1]) + +print("\n%d passed, %d failed" % (ok, fail)) +sys.exit(1 if fail else 0) diff --git a/modules/cachedb_perf/bench/pullsoak/test_relax.py b/modules/cachedb_perf/bench/pullsoak/test_relax.py new file mode 100644 index 00000000000..5d3877a9209 --- /dev/null +++ b/modules/cachedb_perf/bench/pullsoak/test_relax.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""TEST-ONLY: let the requester put an out-of-spec key on the wire. + +A well-behaved node cannot produce the packet this test needs - its own +gate stops it - so the gate is relaxed here, and here only. The slot copy +is still clamped to the production array size, so the SENDER stays sound +and the only thing under test is what the RECEIVER does with the key. +Nothing else is touched, and the file is restored from git afterwards. +""" +import re, sys + +p = "/dn/wt-cp15/modules/cachedb_perf/cachedb_perf.c" +s = open(p).read() + +old_gate = """ if (!pcache_pull_enabled(col) || key->len > PCACHE_PULL_MAX_KEY || + col->col_name.len > 63) + return -1;""" +new_gate = """ if (!pcache_pull_enabled(col) || key->len > 8192 || + col->col_name.len > 63) + return -1; /* TEST BUILD: gate relaxed to reach the serve path */""" +assert old_gate in s, "sender gate not found" +s = s.replace(old_gate, new_gate, 1) + +old_cp = """ memcpy(sl->key, key->s, key->len); + sl->klen = key->len;""" +new_cp = """ { /* TEST BUILD: the wire carries the full key, the slot only what fits */ + int cp = key->len > PCACHE_PULL_MAX_KEY ? PCACHE_PULL_MAX_KEY : key->len; + memcpy(sl->key, key->s, cp); + sl->klen = cp; + }""" +assert old_cp in s, "slot copy not found" +s = s.replace(old_cp, new_cp, 1) + +open(p, "w").write(s) +print("relaxed the sender gate (test build only)") diff --git a/modules/cachedb_perf/bench/queue.c b/modules/cachedb_perf/bench/queue.c new file mode 100644 index 00000000000..e9426dbf90a --- /dev/null +++ b/modules/cachedb_perf/bench/queue.c @@ -0,0 +1,161 @@ +/* + * Queued write path (RabbitMQ-shaped): producers enqueue, consumers apply. + * + * Fair comparison: a FIXED budget of 8 threads, split between producers and + * consumers, versus 8 threads writing directly. What is counted is APPLIED + * writes - work that actually landed in the table - not enqueue rate, because + * an enqueue that never gets applied is not a write. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned core_hash(const str *s1,const str *s2,const unsigned size) +{ char *p,*end; register unsigned v; register unsigned h=0; + end=s1->s+s1->len; + for(p=s1->s;p<=(end-4);p+=4){v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3];ch_h_inc;} + v=0; for(;p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; } + +#define NKEYS 50000 +#define KLEN 16 +#define BNB 16384 +#define BSLOTS 6 +#define TOTALTHR 8 +#define RINGSZ 8192 /* entries per producer ring */ +#define SECS 2 + +typedef struct __attribute__((aligned(64))) { + volatile unsigned version; + volatile unsigned lock; + unsigned char tags[BSLOTS]; + unsigned short used; + void *slot[BSLOTS]; +} bucket; +static bucket *T; + +/* one SPSC ring per producer; consumers each own a disjoint set of rings */ +typedef struct { unsigned hash; unsigned ki; } item; +typedef struct __attribute__((aligned(64))) { + _Alignas(64) volatile unsigned long head; /* producer writes */ + _Alignas(64) volatile unsigned long tail; /* consumer writes */ + item slot[RINGSZ]; +} ring; +static ring *rings; + +static char (*keys)[20]; +static volatile int go, stop; +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static inline void spin_lock(volatile unsigned *l){ while(__sync_lock_test_and_set(l,1)) while(*l) __builtin_ia32_pause(); } +static inline void spin_unlock(volatile unsigned *l){ __sync_lock_release(l); } + +static inline void apply_write(unsigned h) +{ + unsigned b = h & (BNB-1); + spin_lock(&T[b].lock); + __atomic_add_fetch(&T[b].version, 1, __ATOMIC_RELEASE); + T[b].tags[h % BSLOTS] = (unsigned char)(h>>24)|1; + __atomic_add_fetch(&T[b].version, 1, __ATOMIC_RELEASE); + spin_unlock(&T[b].lock); +} + +struct arg { int id, role, nprod, ncons; unsigned long applied, enq, full; }; + +static void *runner(void *p) +{ + struct arg *a = p; + unsigned seed = 4242 + a->id*7919; + unsigned long applied=0, enq=0, full=0; + + while (!go) __builtin_ia32_pause(); + + if (a->role == 0) { /* direct writer */ + while (!stop) for (int r=0;r<256;r++){ + seed=seed*1103515245u+12345u; + int ki=(seed>>8)%NKEYS; str k={keys[ki],KLEN}; + apply_write(core_hash(&k,NULL,0)); applied++; + } + } else if (a->role == 1) { /* producer */ + ring *R = &rings[a->id]; + while (!stop) for (int r=0;r<256;r++){ + seed=seed*1103515245u+12345u; + int ki=(seed>>8)%NKEYS; str k={keys[ki],KLEN}; + unsigned long h_ = R->head, t_ = __atomic_load_n(&R->tail,__ATOMIC_ACQUIRE); + if (h_ - t_ >= RINGSZ) { full++; continue; } /* ring full: back-pressure */ + R->slot[h_ & (RINGSZ-1)].hash = core_hash(&k,NULL,0); + __atomic_store_n(&R->head, h_+1, __ATOMIC_RELEASE); + enq++; + } + } else { /* consumer: drain its share of rings */ + int c = a->id - a->nprod; + while (!stop) { + for (int q = c; q < a->nprod; q += a->ncons) { + ring *R = &rings[q]; + unsigned long t_ = R->tail, h_ = __atomic_load_n(&R->head,__ATOMIC_ACQUIRE); + int n = (int)(h_ - t_); if (n > 64) n = 64; + for (int i=0;islot[(t_+i) & (RINGSZ-1)].hash); + applied++; + } + if (n) __atomic_store_n(&R->tail, t_+n, __ATOMIC_RELEASE); + } + } + } + a->applied=applied; a->enq=enq; a->full=full; + return NULL; +} + +static void run(int nprod, int ncons, double *applied_mops, double *enq_mops, double *fullpct) +{ + pthread_t th[TOTALTHR]; struct arg ar[TOTALTHR]; + int n = nprod + ncons; + for (int i=0;i +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned core_hash(const str *s1,const str *s2,const unsigned size) +{ char *p,*end; register unsigned v; register unsigned h=0; + end=s1->s+s1->len; + for(p=s1->s;p<=(end-4);p+=4){v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3];ch_h_inc;} + v=0; for(;p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; } + +#define NKEYS 50000 +#define VALLEN 200 +#define KLEN 16 +#define SECS 1 +#define BSLOTS 6 +#define BNB 16384 + +typedef struct { unsigned short klen; unsigned vlen; volatile unsigned expires; + char *val; char key[]; } brec; +typedef struct __attribute__((aligned(64))) { + volatile unsigned version; + volatile unsigned lock; + unsigned char tags[BSLOTS]; + unsigned char used; + unsigned char _pad; + brec * volatile slot[BSLOTS]; +} bbucket; + +static bbucket *B; +static char (*keys)[20]; +static unsigned khash[NKEYS]; +static brec *rec0[NKEYS], *rec1[NKEYS]; /* shadow pair for Q swaps */ +static volatile int go, stop; + +/* hot-bucket mode: the few keys landing in one chosen bucket */ +static int hotkeys[64], nhot; + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static inline void spin_lock(volatile unsigned *l){ while(__sync_lock_test_and_set(l,1)) while(*l) __builtin_ia32_pause(); } +static inline void spin_unlock(volatile unsigned *l){ __sync_lock_release(l); } + +struct arg { int id, design, wpct, hot, rewrite_shift; unsigned long ops, retries, reads; }; + +static void *worker(void *p) +{ + struct arg *a = p; + unsigned seed = 12345 + a->id * 7919; + unsigned long ops = 0, retries = 0, reads = 0; + char vbuf[VALLEN] = {0}; + + while (!go) __builtin_ia32_pause(); + + while (!stop) { + for (int rep = 0; rep < 512; rep++) { + seed = seed * 1103515245u + 12345u; + int ki = a->hot ? hotkeys[(seed >> 8) % nhot] : (int)((seed >> 8) % NKEYS); + int isw = a->wpct && ((seed >> 3) % 100) < (unsigned)a->wpct; + int isrw = isw && (((seed >> 13) & 7) == 0); /* 1/8 of writes rewrite value */ + unsigned h = khash[ki]; + unsigned b = h & (BNB - 1); + unsigned char tag = (unsigned char)(h >> 24); if (!tag) tag = 1; + + if (!isw) { + reads++; + if (a->design == 2) { /* Q: no version at all */ + for (int s = 0; s < BSLOTS; s++) { + if (B[b].tags[s] != tag) continue; + brec *r = __atomic_load_n(&B[b].slot[s], __ATOMIC_ACQUIRE); + if (!r || r->klen != KLEN) continue; + if (memcmp(r->key, keys[ki], KLEN) == 0) { + memcpy(vbuf, r->val, 8); + (void)r->expires; + break; + } + } + } else { /* S/H: seqlock */ + unsigned v1, v2; + do { + v1 = __atomic_load_n(&B[b].version, __ATOMIC_ACQUIRE); + if (v1 & 1) { __builtin_ia32_pause(); retries++; continue; } + for (int s = 0; s < BSLOTS; s++) { + if (B[b].tags[s] != tag) continue; + brec *r = B[b].slot[s]; + if (!r || r->klen != KLEN) continue; + if (memcmp(r->key, keys[ki], KLEN) == 0) { memcpy(vbuf, r->val, 8); break; } + } + __atomic_thread_fence(__ATOMIC_ACQUIRE); + v2 = __atomic_load_n(&B[b].version, __ATOMIC_RELAXED); + if (v1 != v2) retries++; + } while (v1 != v2 || (v1 & 1)); + } + } else { + spin_lock(&B[b].lock); + /* find our slot (writer-side scan, as concur.c does) */ + int s; + brec *r = NULL; + for (s = 0; s < BSLOTS; s++) + if (B[b].tags[s] == tag && B[b].slot[s] && + memcmp(B[b].slot[s]->key, keys[ki], KLEN) == 0) { r = B[b].slot[s]; break; } + if (r) { + if (!isrw) { /* TTL bump */ + if (a->design == 0) { /* S: full protocol */ + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + r->expires = (unsigned)ops; + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + } else { /* H/Q: atomic store, no bumps */ + __atomic_store_n(&r->expires, (unsigned)ops, __ATOMIC_RELAXED); + } + } else { /* value rewrite */ + if (a->design == 2) { /* Q: shadow + ptr swap */ + brec *other = (r == rec0[ki]) ? rec1[ki] : rec0[ki]; + memcpy(other->val, vbuf, 8); + memset(other->val + 8, (int)seed, VALLEN - 8); + other->expires = (unsigned)ops; + __atomic_store_n(&B[b].slot[s], other, __ATOMIC_RELEASE); + } else { /* S/H: in-place under version */ + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + memset(r->val, (int)seed, VALLEN); + r->expires = (unsigned)ops; + __atomic_add_fetch(&B[b].version, 1, __ATOMIC_RELEASE); + } + } + } + spin_unlock(&B[b].lock); + } + ops++; + } + } + a->ops = ops; a->retries = retries; a->reads = reads; + return NULL; +} + +static void run(int nthr, int design, int wpct, int hot, double *mops, double *ret1k) +{ + pthread_t th[16]; struct arg ar[16]; + go = stop = 0; + for (int i = 0; i < nthr; i++) { + ar[i] = (struct arg){ i, design, wpct, hot, 3, 0, 0, 0 }; + pthread_create(&th[i], NULL, worker, &ar[i]); + } + double t0 = now(); go = 1; + struct timespec ts = { SECS, 0 }; nanosleep(&ts, NULL); + stop = 1; + unsigned long tot = 0, retr = 0, rds = 0; + for (int i = 0; i < nthr; i++) { pthread_join(th[i], NULL); tot += ar[i].ops; retr += ar[i].retries; rds += ar[i].reads; } + *mops = tot / (now() - t0) / 1e6; + *ret1k = rds ? 1000.0 * retr / rds : 0; +} + +int main(void) +{ + keys = malloc(NKEYS * 20); + for (int i = 0; i < NKEYS; i++) sprintf(keys[i], "%016x", (unsigned)(i * 2654435761u)); + + B = aligned_alloc(64, BNB * sizeof *B); memset(B, 0, BNB * sizeof *B); + printf("sizeof(bbucket) = %zu (want 64)\n", sizeof(bbucket)); + + int placed = 0; + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], KLEN }; + unsigned h = core_hash(&k, NULL, 0); + khash[i] = h; + for (int v = 0; v < 2; v++) { + brec *r = malloc(sizeof(brec) + KLEN + VALLEN); + r->klen = KLEN; r->vlen = VALLEN; r->expires = 0; + memcpy(r->key, keys[i], KLEN); r->val = r->key + KLEN; + memset(r->val, 'a' + (i % 26), VALLEN); + if (v) rec1[i] = r; else rec0[i] = r; + } + unsigned bb = h & (BNB - 1); + unsigned char tag = (unsigned char)(h >> 24); if (!tag) tag = 1; + for (int s = 0; s < BSLOTS; s++) + if (!B[bb].slot[s]) { B[bb].slot[s] = rec0[i]; B[bb].tags[s] = tag; + B[bb].used++; placed++; break; } + } + /* hot-bucket key list: everything that landed where key 0 did */ + unsigned tb = khash[0] & (BNB - 1); + for (int i = 0; i < NKEYS && nhot < 64; i++) + if ((khash[i] & (BNB - 1)) == tb) hotkeys[nhot++] = i; + printf("%d/%d keys placed, hot bucket holds %d keys\n\n", placed, NKEYS, nhot); + + const char *dn[] = { "S seqlock", "H hybrid", "Q qsbr" }; + struct { const char *name; int wpct, hot; } mixes[] = { + { "100% read, uniform", 0, 0 }, + { "95/5 r/w, uniform (bump-heavy)", 5, 0 }, + { "50/50 r/w, ONE hot bucket", 50, 1 }, + }; + int thr[] = { 1, 2, 4, 8 }; + + for (unsigned m = 0; m < 3; m++) { + printf("== %s ==\n", mixes[m].name); + printf("%-8s", "threads"); + for (int d = 0; d < 3; d++) printf(" %14s", dn[d]); + printf(" %s\n", "S retries/1k reads"); + for (unsigned t = 0; t < 4; t++) { + printf("%-8d", thr[t]); + double sret = 0; + for (int d = 0; d < 3; d++) { + double mo, rk; + run(thr[t], d, mixes[m].wpct, mixes[m].hot, &mo, &rk); + if (d == 0) sret = rk; + printf(" %11.2f M/s", mo); + } + printf(" %.3f\n", sret); + } + printf("\n"); + } + return 0; +} diff --git a/modules/cachedb_perf/bench/structs.c b/modules/cachedb_perf/bench/structs.c new file mode 100644 index 00000000000..f2e6a6a859a --- /dev/null +++ b/modules/cachedb_perf/bench/structs.c @@ -0,0 +1,209 @@ +/* + * cachedb_local index-structure shootout. + * 50k keys, 200-byte out-of-line values, allocations scattered to mimic shm + * fragmentation after a busy run. Measures SUCCESSFUL point lookups (hot path). + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; + +#define ch_h_inc h+=v^(v>>3) +static inline unsigned int core_hash(const str *s1, const str *s2, const unsigned int size) +{ + char *p, *end; register unsigned v; register unsigned h = 0; + end=s1->s+s1->len; + for ( p=s1->s ; p<=(end-4) ; p+=4 ){ v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3]; ch_h_inc; } + v=0; for (; p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; +} + +#define NKEYS 50000 +#define VALLEN 200 +#define ITERS 1000000 + +static char keys[NKEYS][20]; +static int klen; + +static double now(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+1e-9*t.tv_nsec; } +static void *salloc(size_t sz){ void *p = malloc(sz); (void)!malloc(16+(rand()&127)); return p; } + +/* ============ A: current - chained list, str pointers, strncmp ============ */ +typedef struct centry { str attr, value; unsigned expires, ttl; int synced; struct centry *next; } centry; +typedef struct { centry *e; int lock; } cbucket; + +/* ============ B: chained, hash cached in node ============ */ +typedef struct hentry { struct hentry *next; unsigned hash; unsigned short klen; char *val; char key[]; } hentry; +typedef struct { hentry *e; int lock; } hbucket; + +/* ============ C: sorted array per bucket (binary search, contiguous) ============ */ +typedef struct { unsigned hash; void *rec; } slot; +typedef struct { slot *v; int n, cap; int lock; } sbucket; +typedef struct { unsigned short klen; char *val; char key[]; } srec; + +/* ============ D: cache-line bucket, 4 inline (hash,ptr) + overflow ============ */ +typedef struct dovf { struct dovf *next; unsigned h[4]; void *p[4]; int n; } dovf; +typedef struct { unsigned h[4]; void *p[4]; int n; dovf *ovf; } dbucket; /* 64B-ish */ + +/* ============ E: flat open addressing, linear probe (Swiss-lite) ============ */ +typedef struct { unsigned hash; void *rec; } fslot; /* hash==0 means empty */ + +int main(void) +{ + srand(12345); + for (int i = 0; i < NKEYS; i++) sprintf(keys[i], "%016x", (unsigned)(i * 2654435761u)); + klen = strlen(keys[0]); + char *val = malloc(VALLEN); + + printf("50000 keys, %d-byte keys, %d-byte values, %d lookups each\n\n", klen, VALLEN, ITERS); + printf("%-46s %10s %9s %10s\n", "design", "ns/lookup", "speedup", "index MB"); + printf("%-46s %10s %9s %10s\n", "------", "---------", "-------", "--------"); + + double base512 = 0, base64k = 0; + + for (int si = 0; si < 2; si++) { + int nb = si ? 65536 : 512; + + /* ---------- A ---------- */ + cbucket *A = calloc(nb, sizeof *A); + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], klen }; + unsigned b = core_hash(&k, NULL, nb); + centry *e = salloc(sizeof(centry) + klen + VALLEN); + memset(e, 0, sizeof *e); + e->attr.s = (char*)e + sizeof(centry); memcpy(e->attr.s, k.s, klen); e->attr.len = klen; + e->value.s = e->attr.s + klen; e->value.len = VALLEN; + e->next = A[b].e; A[b].e = e; + } + double t = now(); volatile unsigned long hit = 0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i*7919)%NKEYS); + str k = { keys[ki], klen }; + unsigned b = core_hash(&k, NULL, nb); + for (centry *e = A[b].e; e; e = e->next) + if (e->attr.len == klen && strncmp(e->attr.s, k.s, klen)==0) { hit++; break; } + } + double ta = (now()-t)*1e9/ITERS; + double memA = (double)nb*sizeof(cbucket)/1048576.0; + if (si) base64k = ta; else base512 = ta; + printf("\n-- %d buckets (load factor %.1f) --\n", nb, (double)NKEYS/nb); + printf("%-46s %10.1f %9s %10.2f\n", "A current: chained list + strncmp", ta, "1.00x", memA); + + /* ---------- B ---------- */ + hbucket *B = calloc(nb, sizeof *B); + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], klen }; + unsigned h = core_hash(&k, NULL, 0); unsigned b = h & (nb-1); + hentry *e = salloc(sizeof(hentry) + klen + VALLEN); + e->hash=h; e->klen=klen; memcpy(e->key,k.s,klen); e->val=e->key+klen; + e->next=B[b].e; B[b].e=e; + } + t = now(); hit = 0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i*7919)%NKEYS); + str k = { keys[ki], klen }; + unsigned h = core_hash(&k,NULL,0); unsigned b = h&(nb-1); + for (hentry *e = B[b].e; e; e = e->next) + if (e->hash==h && e->klen==klen && memcmp(e->key,k.s,klen)==0){hit++;break;} + } + double tb = (now()-t)*1e9/ITERS; + printf("%-46s %10.1f %8.2fx %10.2f\n", "B chained + hash cached in node", tb, ta/tb, + (double)nb*sizeof(hbucket)/1048576.0); + + /* ---------- C ---------- */ + sbucket *C = calloc(nb, sizeof *C); + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], klen }; + unsigned h = core_hash(&k,NULL,0); unsigned b = h&(nb-1); + srec *r = salloc(sizeof(srec)+klen+VALLEN); + r->klen=klen; memcpy(r->key,k.s,klen); r->val=r->key+klen; + if (C[b].n==C[b].cap){ C[b].cap = C[b].cap?C[b].cap*2:4; C[b].v=realloc(C[b].v,C[b].cap*sizeof(slot)); } + int j=C[b].n-1; while(j>=0 && C[b].v[j].hash>h){C[b].v[j+1]=C[b].v[j];j--;} + C[b].v[j+1].hash=h; C[b].v[j+1].rec=r; C[b].n++; + } + t = now(); hit = 0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i*7919)%NKEYS); + str k = { keys[ki], klen }; + unsigned h = core_hash(&k,NULL,0); unsigned b = h&(nb-1); + int lo=0, hi2=C[b].n-1; + while(lo<=hi2){ int m=(lo+hi2)/2; + if(C[b].v[m].hashh) hi2=m-1; + else { srec *r=C[b].v[m].rec; if(r->klen==klen&&memcmp(r->key,k.s,klen)==0)hit++; break; } } + } + double tc = (now()-t)*1e9/ITERS; + printf("%-46s %10.1f %8.2fx %10.2f\n", "C sorted array/bucket + binary search", tc, ta/tc, + (double)nb*sizeof(sbucket)/1048576.0 + (double)NKEYS*sizeof(slot)/1048576.0); + + free(A); free(B); free(C); + } + + /* ---------- D: cache-line buckets, sized so ~3 entries fit inline ---------- */ + { + int nb = 16384; + dbucket *D = calloc(nb, sizeof *D); + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], klen }; + unsigned h = core_hash(&k,NULL,0); unsigned b = h&(nb-1); + srec *r = salloc(sizeof(srec)+klen+VALLEN); + r->klen=klen; memcpy(r->key,k.s,klen); r->val=r->key+klen; + if (D[b].n<4){ D[b].h[D[b].n]=h; D[b].p[D[b].n]=r; D[b].n++; } + else { dovf *o=D[b].ovf; if(!o||o->n==4){ dovf *n2=salloc(sizeof(dovf)); memset(n2,0,sizeof *n2); + n2->next=D[b].ovf; D[b].ovf=n2; o=n2; } + o->h[o->n]=h; o->p[o->n]=r; o->n++; } + } + double t = now(); volatile unsigned long hit=0; + for (int i = 0; i < ITERS; i++) { + int ki = (int)(((long)i*7919)%NKEYS); + str k = { keys[ki], klen }; + unsigned h = core_hash(&k,NULL,0); unsigned b = h&(nb-1); + int found=0; + for(int j=0;jklen==klen&&memcmp(r->key,k.s,klen)==0){hit++;found=1;break;} } + if(!found) for(dovf*o=D[b].ovf;o&&!found;o=o->next) + for(int j=0;jn;j++) if(o->h[j]==h){ srec*r=o->p[j]; + if(r->klen==klen&&memcmp(r->key,k.s,klen)==0){hit++;found=1;break;} } + } + double td=(now()-t)*1e9/ITERS; + printf("\n-- alternative layouts, self-sized --\n"); + printf("%-46s %10.1f %8.2fx %10.2f\n","D 64B cache-line bucket, 4 inline slots",td,base512/td, + (double)nb*sizeof(dbucket)/1048576.0); + printf("%-46s %10s %8.2fx %10s\n"," (vs well-sized chained, B@64k)","", base64k/td, ""); + } + + /* ---------- E: flat open addressing, load 0.38 ---------- */ + { + int ns = 131072; + fslot *E = calloc(ns, sizeof *E); + for (int i = 0; i < NKEYS; i++) { + str k = { keys[i], klen }; + unsigned h = core_hash(&k,NULL,0); if(!h)h=1; + srec *r = salloc(sizeof(srec)+klen+VALLEN); + r->klen=klen; memcpy(r->key,k.s,klen); r->val=r->key+klen; + unsigned p = h&(ns-1); + while(E[p].hash) p=(p+1)&(ns-1); + E[p].hash=h; E[p].rec=r; + } + double t=now(); volatile unsigned long hit=0; + for (int i = 0; i < ITERS; i++) { + int ki=(int)(((long)i*7919)%NKEYS); + str k={keys[ki],klen}; + unsigned h=core_hash(&k,NULL,0); if(!h)h=1; + unsigned p=h&(ns-1); + while(E[p].hash){ if(E[p].hash==h){ srec*r=E[p].rec; + if(r->klen==klen&&memcmp(r->key,k.s,klen)==0){hit++;break;} } + p=(p+1)&(ns-1); } + } + double te=(now()-t)*1e9/ITERS; + printf("%-46s %10.1f %8.2fx %10.2f\n","E flat open addressing (linear probe)",te,base512/te, + (double)ns*sizeof(fslot)/1048576.0); + printf("%-46s %10s %8.2fx %10s\n"," (vs well-sized chained, B@64k)","", base64k/te, ""); + } + free(val); + return 0; +} diff --git a/modules/cachedb_perf/bench/warmup.c b/modules/cachedb_perf/bench/warmup.c new file mode 100644 index 00000000000..7ef7610d48c --- /dev/null +++ b/modules/cachedb_perf/bench/warmup.c @@ -0,0 +1,95 @@ +/* + * Two costs OpenSIPS shm currently pays, neither addressed: + * + * 1. FIRST TOUCH. mmap(MAP_ANON) is demand-paged, so the first write to each + * 4K page takes a minor fault. This is what pre-warming would remove. + * 2. TLB PRESSURE. A large cache accessed randomly through 4K pages misses + * the TLB on nearly every access. Huge pages would remove that - and it is + * an ONGOING cost, not a one-time one. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +#define REGION (256UL*1024*1024) +#define ENTSZ 256 +#define NENT (REGION/ENTSZ) + +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} + +static void *map_region(int huge) +{ + void *p = mmap(NULL, REGION, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS, -1, 0); /* exactly what shm_getmem does */ + if (p == MAP_FAILED) { perror("mmap"); exit(1); } +#ifdef MADV_HUGEPAGE + madvise(p, REGION, huge ? MADV_HUGEPAGE : MADV_NOHUGEPAGE); +#endif + return p; +} + +int main(void) +{ + char payload[ENTSZ]; + memset(payload, 'x', sizeof payload); + printf("region %lu MB, entry %d B, %lu entries, page %ld B\n\n", + REGION/1024/1024, ENTSZ, NENT, sysconf(_SC_PAGESIZE)); + + /* ---------- 1. first-touch cost ---------- */ + printf("== cost of the FIRST write to each page (what pre-warming removes) ==\n"); + + char *cold = map_region(0); + double t = now(); + for (unsigned long i = 0; i < NENT; i++) memcpy(cold + i*ENTSZ, payload, ENTSZ); + double t_cold = now() - t; + + char *warm = map_region(0); + t = now(); + memset(warm, 0, REGION); /* the proposed warm-up */ + double t_warm = now() - t; + t = now(); + for (unsigned long i = 0; i < NENT; i++) memcpy(warm + i*ENTSZ, payload, ENTSZ); + double t_pre = now() - t; + + printf(" fill cold (faults inline) %8.1f ms %6.1f ns/entry\n", t_cold*1e3, t_cold*1e9/NENT); + printf(" fill pre-warmed %8.1f ms %6.1f ns/entry\n", t_pre*1e3, t_pre*1e9/NENT); + printf(" warm-up pass itself %8.1f ms (one-time, %.0f MB memset)\n", + t_warm*1e3, (double)REGION/1024/1024); + printf(" -> first touch costs %.1f ns/entry, %.0f ms total for %lu MB\n", + (t_cold-t_pre)*1e9/NENT, (t_cold-t_pre)*1e3, REGION/1024/1024); + printf(" -> net saving of pre-warming: %.0f ms (it moves cost, it does not remove it)\n\n", + (t_cold - t_pre - t_warm)*1e3); + munmap(cold, REGION); + + /* ---------- 2. ongoing TLB cost: 4K vs huge pages ---------- */ + printf("== ONGOING cost: random access, 4K pages vs transparent huge pages ==\n"); + char *h4 = map_region(0), *h2 = map_region(1); + memset(h4, 1, REGION); memset(h2, 1, REGION); + + unsigned long n = 20000000; + volatile unsigned long sink = 0; + for (int pass = 0; pass < 2; pass++) { + char *r = pass ? h2 : h4; + unsigned seed = 12345; + t = now(); + for (unsigned long i = 0; i < n; i++) { + seed = seed*1103515245u + 12345u; + unsigned long off = ((unsigned long)seed % NENT) * ENTSZ; + sink += r[off]; /* one random line per iteration */ + } + double el = now() - t; + printf(" %-28s %7.1f ns/access %6.1f M/s\n", + pass ? "transparent huge pages" : "4K pages (as today)", + el*1e9/n, n/el/1e6); + } + printf(" (sink=%lu)\n", sink); + + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/enabled","r"); + if (f) { char b[128]; if (fgets(b,sizeof b,f)) printf("\n system THP setting: %s", b); fclose(f); } + return 0; +} diff --git a/modules/cachedb_perf/bench/wbuf.c b/modules/cachedb_perf/bench/wbuf.c new file mode 100644 index 00000000000..523392c7c79 --- /dev/null +++ b/modules/cachedb_perf/bench/wbuf.c @@ -0,0 +1,174 @@ +/* + * Does a write-staging buffer (LSM memtable) help cachedb_perf? + * + * WRITE PATH, 1..8 threads: + * A per-bucket lock (the current design) + * B shared 1MB append buffer, atomic fetch_add on one head <- "no lock" + * C per-process append buffer, no atomic at all + * + * READ PATH: what it costs if a reader must also consult the buffers. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned core_hash(const str *s1,const str *s2,const unsigned size) +{ char *p,*end; register unsigned v; register unsigned h=0; + end=s1->s+s1->len; + for(p=s1->s;p<=(end-4);p+=4){v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3];ch_h_inc;} + v=0; for(;p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; } + +#define NKEYS 50000 +#define KLEN 16 +#define VALLEN 200 +#define ENTSZ (KLEN + VALLEN + 16) /* what one staged write costs */ +#define BUFSZ (1024*1024) /* the proposed 1 MB */ +#define BNB 16384 +#define BSLOTS 6 +#define SECS 2 +#define MAXTHR 8 + +typedef struct { unsigned short klen; unsigned vlen; char *val; char key[]; } brec; +typedef struct __attribute__((aligned(64))) { + volatile unsigned version; + volatile unsigned lock; + unsigned char tags[BSLOTS]; + unsigned short used; + brec *slot[BSLOTS]; +} bucket; +static bucket *T; + +/* B: one shared buffer, one shared head */ +static struct { _Alignas(64) volatile unsigned long head; char data[BUFSZ]; } shared_buf; + +/* C: per-thread buffers, each with its own head */ +static struct { _Alignas(64) unsigned long head; char data[BUFSZ/MAXTHR]; } priv_buf[MAXTHR]; + +static char (*keys)[20]; +static volatile int go, stop; +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static inline void spin_lock(volatile unsigned *l){ while(__sync_lock_test_and_set(l,1)) while(*l) __builtin_ia32_pause(); } +static inline void spin_unlock(volatile unsigned *l){ __sync_lock_release(l); } + +struct arg { int id, design; unsigned long ops, wrapped; }; + +static void *writer(void *p) +{ + struct arg *a = p; + unsigned seed = 999 + a->id*7919; + unsigned long ops = 0, wrapped = 0; + char payload[ENTSZ]; + memset(payload, 'x', sizeof payload); + + while (!go) __builtin_ia32_pause(); + while (!stop) { + for (int rep = 0; rep < 256; rep++) { + seed = seed*1103515245u + 12345u; + int ki = (seed >> 8) % NKEYS; + str k = { keys[ki], KLEN }; + unsigned h = core_hash(&k, NULL, 0); + + if (a->design == 0) { /* A: per-bucket lock */ + unsigned b = h & (BNB-1); + spin_lock(&T[b].lock); + __atomic_add_fetch(&T[b].version, 1, __ATOMIC_RELEASE); + T[b].tags[h % BSLOTS] = (unsigned char)(h>>24)|1; /* mutate */ + __atomic_add_fetch(&T[b].version, 1, __ATOMIC_RELEASE); + spin_unlock(&T[b].lock); + } else if (a->design == 1) { /* B: shared buffer, atomic head */ + unsigned long off = __atomic_fetch_add(&shared_buf.head, ENTSZ, __ATOMIC_RELAXED); + if (off + ENTSZ > BUFSZ) { /* full: reorganiser must drain */ + __atomic_store_n(&shared_buf.head, 0, __ATOMIC_RELAXED); + wrapped++; off = 0; + } + memcpy(shared_buf.data + off, payload, ENTSZ); + } else { /* C: private buffer, no atomic */ + unsigned long off = priv_buf[a->id].head; + if (off + ENTSZ > BUFSZ/MAXTHR) { priv_buf[a->id].head = 0; wrapped++; off = 0; } + memcpy(priv_buf[a->id].data + off, payload, ENTSZ); + priv_buf[a->id].head = off + ENTSZ; + } + ops++; + } + } + a->ops = ops; a->wrapped = wrapped; + return NULL; +} + +static double run(int nthr, int design, unsigned long *wrapped) +{ + pthread_t th[MAXTHR]; struct arg ar[MAXTHR]; + go = stop = 0; + for (int i=0;i holds %d writes\n", + ENTSZ, BUFSZ/1024, BUFSZ/ENTSZ); + printf("at 6000 CPS with one write per call, that is %.2f seconds of headroom\n\n", + (double)(BUFSZ/ENTSZ)/6000.0); + + printf("== WRITE throughput (Mops/s) ==\n"); + printf("%-8s %14s %16s %16s\n","threads","A bucket lock","B shared buf","C private buf"); + int thr[]={1,2,4,8}; + double a1=0,b1=0,c1=0; + for (unsigned i=0;i8\n", + "scaling",a/a1,b/b1,c/c1); + } + + /* ---- read penalty: reader must also consult N buffers ---- */ + printf("\n== READ cost when the reader must also check staged writes ==\n"); + brec *r = malloc(sizeof(brec)+KLEN+VALLEN); + r->klen=KLEN; memcpy(r->key,keys[0],KLEN); r->val=r->key+KLEN; + for (int i=0;i>24)|1;break;} + } + /* per-buffer index: one extra hash probe per buffer consulted */ + unsigned *idx[MAXTHR]; + for (int i=0;i>(q&7)) & 4095]; /* buffer index probe */ + unsigned b=h&(BNB-1); + for(int s=0;s>24)|1)){ + brec *rr=T[b].slot[s]; if(rr && rr->klen==KLEN){memcpy(vbuf,rr->val,8);hits++;} break; } + } + double ns=(now()-t)*1e9/3000000; + printf(" table + %d buffer probe(s) %7.1f ns/read %s\n", np, ns, + pi==0 ? "<- baseline (no staging)" : ""); + } + printf(" (hits=%lu)\n", hits); + return 0; +} diff --git a/modules/cachedb_perf/bench/worker.c b/modules/cachedb_perf/bench/worker.c new file mode 100644 index 00000000000..cff74ebc5fb --- /dev/null +++ b/modules/cachedb_perf/bench/worker.c @@ -0,0 +1,135 @@ +/* + * Does moving the sort off the hot path pay? + * Bucket = sorted prefix (binary search) + small unsorted append tail. + * Writers append O(1); a background worker merges tail -> prefix. + * Compared against: current chain, and eager sorted-insert (memmove in hot path). + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +typedef struct { char *s; int len; } str; +#define ch_h_inc h+=v^(v>>3) +static inline unsigned int core_hash(const str *s1, const str *s2, const unsigned int size) +{ + char *p, *end; register unsigned v; register unsigned h = 0; + end=s1->s+s1->len; + for ( p=s1->s ; p<=(end-4) ; p+=4 ){ v=(*p<<24)+(p[1]<<16)+(p[2]<<8)+p[3]; ch_h_inc; } + v=0; for (; p>11))+((h>>13)+(h>>23)); + return size?((h)&(size-1)):h; +} + +#define NKEYS 50000 +#define VALLEN 200 +#define ITERS 1000000 +#define NBUCK 512 /* the pathological default */ +#define TAILMAX 8 + +static char keys[NKEYS][20]; +static int klen; +static double now(void){struct timespec t;clock_gettime(CLOCK_MONOTONIC,&t);return t.tv_sec+1e-9*t.tv_nsec;} +static void *salloc(size_t sz){void*p=malloc(sz);(void)!malloc(16+(rand()&127));return p;} + +typedef struct { unsigned short klen; char *val; char key[]; } rec; +typedef struct { unsigned hash; rec *r; } slot; + +/* eager-sorted bucket */ +typedef struct { slot *v; int n, cap; } sb; +/* prefix + tail bucket */ +typedef struct { slot *v; int n, cap; slot tail[TAILMAX]; int tn; } pb; + +/* current chain */ +typedef struct centry { str attr, value; unsigned e,t; int s; struct centry *next; } centry; + +static rec *mkrec(const char *k){ rec *r=salloc(sizeof(rec)+klen+VALLEN); + r->klen=klen; memcpy(r->key,k,klen); r->val=r->key+klen; return r; } + +static int cmp(const void *a,const void *b){ + unsigned x=((const slot*)a)->hash,y=((const slot*)b)->hash; + return xy?1:0); } + +int main(void) +{ + srand(12345); + for(int i=0;iattr.s=(char*)e+sizeof(centry); memcpy(e->attr.s,k.s,klen); e->attr.len=klen; + e->next=CH[b]; CH[b]=e; } + printf(" chain, prepend %8.1f ns/insert\n",(now()-t)*1e9/NKEYS); + + sb *S=calloc(NBUCK,sizeof *S); + t=now(); + for(int i=0;i=0&&S[b].v[j].hash>h){S[b].v[j+1]=S[b].v[j];j--;} + S[b].v[j+1].hash=h;S[b].v[j+1].r=r;S[b].n++; } + printf(" eager sorted insert (memmove) %8.1f ns/insert\n",(now()-t)*1e9/NKEYS); + + pb *P=calloc(NBUCK,sizeof *P); + long merges=0; double merge_time=0; + t=now(); + for(int i=0;iP[b].cap){P[b].cap=(P[b].n+TAILMAX)*2;P[b].v=realloc(P[b].v,P[b].cap*sizeof(slot));} + memcpy(P[b].v+P[b].n,P[b].tail,TAILMAX*sizeof(slot)); P[b].n+=TAILMAX; P[b].tn=0; + qsort(P[b].v,P[b].n,sizeof(slot),cmp); + merge_time+=now()-m0; merges++; + } + P[b].tail[P[b].tn].hash=h; P[b].tail[P[b].tn].r=r; P[b].tn++; } + double tp=now()-t; + printf(" prefix+tail, append only %8.1f ns/insert (excl. merge: %.1f)\n", + tp*1e9/NKEYS,(tp-merge_time)*1e9/NKEYS); + printf(" -> %ld merges, %.1f ms total, %.1f us each <- worker's job\n\n", + merges,merge_time*1e3,merge_time*1e6/(merges?merges:1)); + + /* ---------------- LOOKUP cost ---------------- */ + printf("== lookup cost ==\n"); + volatile unsigned long hit=0; + + t=now(); + for(int i=0;inext) + if(e->attr.len==klen&&strncmp(e->attr.s,k.s,klen)==0){hit++;break;} } + double lc=(now()-t)*1e9/ITERS; + printf(" chain + strncmp (current) %8.1f ns 1.00x\n",lc); + + t=now(); hit=0; + for(int i=0;i>1; + if(S[b].v[m].hashh)hi=m-1; + else{rec*r=S[b].v[m].r; if(r->klen==klen&&memcmp(r->key,k.s,klen)==0)hit++; break;}} } + double ls=(now()-t)*1e9/ITERS; + printf(" fully sorted, binary search %8.1f ns %5.1fx\n",ls,lc/ls); + + t=now(); hit=0; + for(int i=0;i>1; + if(P[b].v[m].hashh)hi=m-1; + else{rec*r=P[b].v[m].r; if(r->klen==klen&&memcmp(r->key,k.s,klen)==0){hit++;found=1;} break;}} + if(!found) for(int j=0;jklen==klen&&memcmp(r->key,k.s,klen)==0){hit++;break;} } } + double lp=(now()-t)*1e9/ITERS; + printf(" prefix (bsearch) + tail (scan <=%d) %8.1f ns %5.1fx\n",TAILMAX,lp,lc/lp); + return 0; +} diff --git a/modules/cachedb_perf/cachedb_perf.c b/modules/cachedb_perf/cachedb_perf.c new file mode 100644 index 00000000000..533ad735caf --- /dev/null +++ b/modules/cachedb_perf/cachedb_perf.c @@ -0,0 +1,5089 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include + +#include "../../sr_module.h" +#include "../../dprint.h" +#include "../../statistics.h" +#include "../../mi/mi.h" +#include "../../mi/item.h" +#include "../../ut.h" +#include "../../pvar.h" +#include "../../timer.h" +#include "../../mem/mem.h" +#include "../../mem/shm_mem.h" +#include "../../lib/csv.h" +#include "../../evi/evi_modules.h" +#include "../../bin_interface.h" +#include +#include +#include "../clusterer/api.h" +#ifdef CLUSTERER_CTRL_SUPPORT +/* Optional at build time: the controller offers an alternative (encrypted + * multicast) transport for pulls, but cachedb_perf must never require it. + * Without this flag every pull and sync rides the clusterer's bin links. */ +#include "../clusterer_controller/api.h" +#endif +#include "pull_api.h" + +#include "cachedb_perf.h" +#include "pcache_mem.h" +#include "pcache_arena.h" +#include "pcache_htable.h" +#include "pcache_db.h" + +str pcache_mod_name = str_init("perf"); + +static int mod_init(void); +static int child_init(int rank); +static void mod_destroy(void); + +pcache_col_t *pcache_collection = NULL; +pcache_url_t *pcache_url_list = NULL; +/* the collection behind the engine's default (groupless) connection - + * what cache_store("perf", ...) writes to; the glob functions default + * to it so both views always agree */ +static pcache_col_t *pcache_default_col = NULL; +static int arena_selftest = 0; +static int htable_selftest = 0; +extern int pcache_arena_hugepage_mb; +static int expiry_sweep_period = 1; /* seconds; 0 disables the sweep */ +/* CP-09 growth: split buckets while entries/nbuckets exceeds this; 0 = off. + * Default 2 keeps load factor low so the 84 ns bucket shape holds at scale - + * the whole reason this module exists (cachedb_local cannot resize). */ +static int growth_load_factor = 2; +static int growth_budget = 4096; /* max splits per maintenance tick */ + +/* ---- CP-11 observability events ---- */ +static str evi_expired_name = str_init("E_CACHEDB_PERF_EXPIRED"); +static str evi_nomem_name = str_init("E_CACHEDB_PERF_NOMEM"); +static str evi_grown_name = str_init("E_CACHEDB_PERF_GROWN"); +static str evi_degraded_name = str_init("E_CACHEDB_PERF_MEM_DEGRADED"); +static event_id_t evi_expired_id = EVI_ERROR; +static event_id_t evi_nomem_id = EVI_ERROR; +static event_id_t evi_grown_id = EVI_ERROR; +static event_id_t evi_degraded_id = EVI_ERROR; +/* event parameter names */ +static str evp_collection = str_init("collection"); +static str evp_key = str_init("key"); +static str evp_size = str_init("size"); +static str evp_buckets = str_init("buckets"); +static str evp_prev_buckets = str_init("prev_buckets"); +static str evp_splits = str_init("splits"); +static str evp_entries = str_init("entries"); +static str evp_tier = str_init("tier"); +static str evp_backing = str_init("backing"); +static str evp_requested_mb = str_init("requested_mb"); +static str evp_overcommit = str_init("overcommit_pages"); +/* CSV of collections opted in to E_CACHEDB_PERF_EXPIRED (per-collection so a + * high-churn collection reaping in bulk pays only if it asked to); "" = none */ +static char *event_expired_collections = NULL; + +/* ---- CP-19 DB persistence ---- */ +static char *db_url = NULL; /* a db_* backend URL */ +static char *db_table = (char *)"cachedb_perf"; +/* 0 = off, 1 = load on startup, 2 = load on startup + save on shutdown */ +static int db_mode = 0; +/* CSV of the collections that auto load/save with db_mode; "" = none */ +static char *persist_collections = NULL; + +/* ---- CP-19 Stage 2: cluster sync (save-then-broadcast, pull-from-DB) ---- */ +static struct clusterer_binds clusterer_api; +static str pcache_sync_cap = str_init("cachedb-perf-sync"); +static int sync_cluster_id = 0; /* modparam; 0 = off */ +static char *sync_shtag_str; /* modparam "name/cluster_id"; failover sync */ +static str pc_shtag; /* parsed tag name */ +static int pc_shtag_cid; /* parsed tag cluster */ + +/* ---- CP-15.5: cross-node pull ---------------------------------------- */ +/* CP-15.5 cross-node pull, on the same capability as the sync packets */ +#define PCACHE_PULL_REQ 2 +#define PCACHE_PULL_RPL 3 + +#define PCACHE_PULL_SLOTS 64 /* concurrent in-flight pulls */ +/* Defaults for the pull_max_value / pull_max_key modparams below. Sized from + * measurement rather than round numbers: the live cachedb_perf collections on + * the billing gateways hold values of 1-20 bytes under keys of at most 33, and + * sql_cacher's are ~70 bytes. 512/128 is roughly 25x and 4x that headroom. + * + * dns_cache is the deliberate exception - it serialises whole record sets and + * runs to several KB with no real bound - which is exactly why these are + * configurable instead of constants. A dns_cache deployment raises + * pull_max_value and accepts a larger slot (hence fewer slots for the same + * memory); everything above the cap already degrades through the existing + * PCACHE_FOUND_OVERSIZE path, so the cost is "not pulled cross-node", never a + * wrong answer. */ +#define PCACHE_PULL_MAX_VAL_DEF 512 +#define PCACHE_PULL_MAX_KEY_DEF 128 +/* Hard ceilings for the runtime caps below. They stay compile-time because + * three per-call scratch buffers are stack arrays and the negative-cache slot + * embeds a key inline - a modparam able to grow those without bound would + * trade a queue limit for a stack overflow. */ +#define PCACHE_PULL_MAX_VAL 8192 +#define PCACHE_PULL_MAX_KEY 256 +#define PCACHE_NEG_SLOTS 256 /* direct-mapped negative cache */ +/* how long past its deadline a woken slot is left for its caller to come + * back and finish() before the reaper takes it away regardless */ +#define PCACHE_PULL_ABANDON_US (5 * 1000000) +/* a peer that answered within this many seconds is treated as answering; + * beyond it we only know it HAS answered at some point, not that it still + * would - which is why the raw counters are reported beside the verdict */ +#define PCACHE_PEER_FRESH_S 300 +#define CL_MAX_NODE_ID 256 /* the cluster stack's design cap */ +static char *pull_transport_str; /* "bin" (default) | "clctr" */ +static int pull_max_value = PCACHE_PULL_MAX_VAL_DEF; +static int pull_max_key = PCACHE_PULL_MAX_KEY_DEF; +/* byte offsets into a slot, computed once from the caps above - the key and + * value buffers are no longer fixed members, so slots are sized at init */ +static int pull_slot_sz; +static int pull_timeout_ms = 50; /* how long a miss waits for peers */ +/* Optional extra bound on how late an answer may be and still be stored. + * + * 0 (default) = no time bound: what makes a late answer valid is that the + * VALUE is still valid, and the value carries its own expiry. A peer reports + * ttl_left in RELATIVE seconds and we add it to our own clock, so nothing + * here needs the cluster's clocks to agree. The practical ceiling is the + * slot's own life, PCACHE_PULL_ABANDON_US. + * + * Set it non-zero only where the script DELETES keys from a replicated + * collection: store-if-absent cannot tell "never had it" from "deleted a + * moment ago", so a peer's copy could resurrect a key the script removed. + * A deployment that only writes and lets TTLs expire cannot hit that. */ +static int pull_linger_ms = 0; +static char *replicate_collections; /* CSV opt-in; nothing pulls by default */ +static int pull_ready; /* transport up AND a collection opted in */ + +/* CP-15.8: the pull may ride the controller's encrypted multicast plane + * instead of the clusterer's TCP mesh. One query becomes one packet + * regardless of cluster size, and it is encrypted - which the BIN links + * are not. Everything above the transport is identical; only how a + * request leaves and a reply comes back changes. */ +#ifdef CLUSTERER_CTRL_SUPPORT +static clctr_api_t clctr_api; +static str pull_channel = str_init("cdbperf-pull"); +#endif +/* stays 0 for the whole run when the controller is not compiled in */ +static int pull_via_clctr; + +/* flat wire format for the controller plane, which carries bytes rather + * than the BIN push/pop stream. All integers network order. + * request: [u8 REQ][u32 id][u8 collen][col][u16 klen][key] + * reply: [u8 RPL][u32 id][u8 found][u32 ttl][u16 klen][key][u16 vlen][val] + * @found: 0 = not here, 1 = value follows, 2 = held but too big to send. */ +#define PCACHE_CLCTR_REQ 1 +#define PCACHE_CLCTR_RPL 2 +/* fixed bytes of each framing, so the size checks and the budget the serve + * path hands out cannot drift from what the writers actually emit */ +#define PCACHE_CLCTR_REQ_HDR 8 /* type + id + collen + klen */ +#define PCACHE_CLCTR_RPL_HDR 14 /* type + id + found + ttl + klen+vlen */ +#define PCACHE_FOUND_NO 0 +#define PCACHE_FOUND_YES 1 +#define PCACHE_FOUND_OVERSIZE 2 + +/* One in-flight pull. The request is issued by whichever process took the + * miss, but the replies land in whichever process the transport delivers + * them to - so the rendezvous has to live in shm, keyed by request id. + * (Today the requester polls this slot; the async work of CP-15.9 replaces + * the poll with an eventfd it registers here, and nothing else changes.) */ +struct pcache_pull_slot { + unsigned int id; /* 0 = free */ + int efd; /* readable once an answer landed */ + /* absolute us, like the negative cache: a pull that never gets a + * conclusive answer has to be reclaimed on time, and second-grained + * ticks would hold a SIP transaction up to a second past a timeout + * the operator set in milliseconds */ + utime_t deadline; + /* the reaper woke this slot; do not keep re-arming the eventfd on + * every tick while the consumer works its way back to finish() */ + int reaped; + unsigned int gen; /* membership generation at dispatch */ + int expect; /* peers we asked */ + int negative; /* peers that answered "not here" */ + /* which nodes have answered, so a repeated reply cannot be counted + * twice - two negatives from one node would otherwise reach @expect + * and manufacture a "nobody has it" that nobody said */ + unsigned char answered[(CL_MAX_NODE_ID + 7) / 8]; + int done; /* 1 = a value landed */ + int oversize; /* a peer HAS it but could not send */ + int hinted; /* asked one node, not the cluster */ + int partial; /* more peers than the snapshot held */ + /* The waiter left without a value and handed this slot to the protocol + * rather than the pool: an answer may still be in flight, and the slot + * holds the only record of which collection and key it belongs to (the + * reply carries neither). A late answer that lands here is stored by + * pcache_pull_do_reply() instead of being dropped. Reclaimed by the + * reaper at deadline + PCACHE_PULL_ABANDON_US, or stolen sooner if the + * pool runs dry. */ + int orphan; + unsigned int expires; /* ABSOLUTE, as the owner holds it */ + unsigned int vlen; + int klen; + char col[64]; + int collen; + /* key[pull_max_key] then val[pull_max_value] follow this header; reach + * them with pull_slot_key()/pull_slot_val(). Kept as a trailing blob + * rather than two fixed arrays so the caps can be configured without + * every slot paying for the largest value anyone might ever store. */ + char buf[]; +}; + +#define pull_slot_key(sl) ((sl)->buf) +#define pull_slot_val(sl) ((sl)->buf + pull_max_key) +#define pull_slot_at(i) ((struct pcache_pull_slot *)((char *)pull_slots \ + + (size_t)(i) * pull_slot_sz)) +static struct pcache_pull_slot *pull_slots; +static gen_lock_t *pull_lock; +static unsigned int *pull_next_id; /* shm: ids must be unique per node */ + +/* pull counters, deliberately separate from hits/misses so a pulled key + * cannot flatter the local hit rate (R6) */ +/* Pull counters. In shm and bumped from several processes - the request + * side runs in whichever worker took the miss, the reply and serve sides in + * whichever one the transport picked - so the increments are atomic. A + * plain ++ would drop counts under exactly the load worth measuring. This + * is one shared line, which the module forbids on the hot path (CP-06); a + * cross-node miss is not the hot path. */ +static unsigned int *pull_stats; /* PULL_ST_* counters */ +/* Rate-limit state for the send-failure warning (pull_send_failed()). In shm + * rather than a plain static because a pull reply goes out from whichever + * worker happened to receive the request: a per-process limiter would let all + * ~30 SIP workers warn once per interval each. */ +static struct pcache_send_warn { + unsigned int last; /* get_ticks() when we last warned */ + unsigned int suppressed; /* failures folded into the next warning */ +} *pull_send_warn, *pull_xcluster_warn; + +/* Negative cache (R4). A key that is genuinely nowhere costs a full + * round of questions, and SIP retransmits ask again a few hundred + * milliseconds later - so remember "nobody had it" just long enough to + * absorb the retransmit, and no longer: the key may legitimately be + * created on another node a second from now, and a negative that outlives + * that turns a transient miss into a hard failure. + * + * Kept out of the cache proper, deliberately: a negative is not a value. + * Putting it in the table would make perf_keys and perf_dump show keys + * that do not exist and would count in the entry total. Direct-mapped, + * so a fresh negative may evict an older one - losing one only costs a + * repeated question. */ +struct pcache_neg_slot { + unsigned int hash; /* 0 = free */ + utime_t deadline; /* absolute us */ + int klen, collen; + char key[PCACHE_PULL_MAX_KEY]; + char col[64]; +}; +static struct pcache_neg_slot *neg_slots; +static gen_lock_t *neg_lock; +static int pull_negative_ms = 300; /* modparam; 0 = no negative cache */ +static int pull_on_miss; /* modparam; read repair on the get path */ +#define PULL_ST_REQUESTED 0 +#define PULL_ST_SERVED 1 +#define PULL_ST_RECEIVED 2 +#define PULL_ST_TIMEOUT 3 +#define PULL_ST_STORED 4 +#define PULL_ST_SUPPRESSED 5 /* asks a cached negative absorbed */ +/* slots the reaper had to release because the caller never collected them - + * distinct from a timeout, which the caller DID collect */ +#define PULL_ST_ABANDONED 6 +/* A pull datagram the transport refused to send. Distinct from a TIMEOUT: + * the request never left this node, so no peer was ever given the chance to + * answer it. */ +#define PULL_ST_SEND_FAIL 7 +/* a waiter left without a value and the slot was kept for a late answer */ +#define PULL_ST_ORPHANED 8 +/* that late answer arrived and was stored - convergence the old code lost */ +#define PULL_ST_LATE_STORED 9 +/* an orphan's slot was reclaimed early because the pool ran dry */ +#define PULL_ST_ORPHAN_EVICTED 10 +/* a late answer arrived but a local write had already filled the key */ +#define PULL_ST_LATE_SUPERSEDED 11 +/* a late answer landed after pull_linger_ms and was refused as too stale */ +#define PULL_ST_LATE_EXPIRED 12 +/* an orphan reached the end of its life with no late answer - the ordinary + * outcome of a timeout, and explicitly NOT an abandoned slot: its caller DID + * collect it, which is the distinction PULL_ST_ABANDONED exists to make */ +#define PULL_ST_ORPHAN_EXPIRED 13 +/* a pull message arrived on a controller cluster this module does not sync on + * and was refused. Non-zero means either a genuine multi-cluster node doing + * the right thing, or sync_cluster_id naming a cluster the controller does not + * manage - the accompanying warning tells the two apart. */ +#define PULL_ST_FOREIGN_CLUSTER 14 +/* A peer answered "I do not have it". Counted per REPLY, so on a cluster + * larger than two a single request can raise this more than once - the + * requested/answered identity below is exact only for a 2-node cluster. + * Distinct from PULL_ST_SUPPRESSED, which counts the SECOND and later asks + * absorbed by an already-cached negative (pull_negative_ms): without this + * counter the FIRST negative was invisible, and + * `pulls_requested - pulls_received` could not be explained from statistics + * at all - "the peer genuinely did not have it" looked identical to "the + * request was swallowed". */ +#define PULL_ST_NEGATIVE 15 +/* A peer HAS the key but it is too big for the cluster transport. Also not a + * value and also not an absence, so it needs its own counter for the same + * reason. */ +#define PULL_ST_OVERSIZE 16 +/* A miss that reached the pull gate and was refused before any request left + * this node. Without these, `misses` and `pulls_requested` could not be + * reconciled at all: on a live gateway ~986 misses a minute were counted by the + * cache and then vanished, which made it impossible to answer the only question + * that matters about pull_on_miss - is it doing anything for this collection? + * + * The reasons are kept apart because they call for completely different + * actions: NOTREPLICATED is a configuration statement (this collection was + * never meant to pull), NOPEERS means the cluster is not formed, NOSLOT means + * demand is outrunning the slot table, and TOOLONG means the key can never be + * asked for at all. Rolled into one counter they would be indistinguishable, + * which is how the original hole came to be. + * + * Together with PULL_ST_SUPPRESSED these close the miss side the way + * PULL_ST_NEGATIVE closed the reply side: + * misses == requested + suppressed + skipped(all four) */ +#define PULL_ST_SKIP_NOTREPLICATED 17 /* pull off for this collection */ +#define PULL_ST_SKIP_TOOLONG 18 /* key/collection name cannot be asked */ +#define PULL_ST_SKIP_NOPEERS 19 /* no live cluster member to ask */ +#define PULL_ST_SKIP_NOSLOT 20 /* slot table full, nothing evictable */ +#define PULL_ST_MAX 21 +/* Two different readinesses, deliberately kept apart: + * cluster_ready - the clusterer is bound, the capability is registered and + * membership is being tracked. Everything cross-node needs + * this and nothing more. + * sync_ready - that, plus a DB to snapshot through. Only perf_sync and + * the failover hook need it, because only they use the DB. + * Conflating them made a cache that only ever pulls demand a database it + * never touches. */ +static int cluster_ready = 0; +static int sync_ready = 0; /* cluster_ready + a usable db_url */ + +/* Cluster membership view (CP-15.4). The clusterer node list changes at + * runtime (under clusterer_controller, on every join/leave/eviction), so + * anything that fans work out to peers must snapshot the member set and + * notice when it changed mid-flight. The event callback below maintains + * this shm view; `generation` is the load-bearing field - a future + * cross-node pull snapshots it together with its responder set and + * re-checks it on completion, because an absence conclusion drawn across + * a membership change is unsafe. Counters are monitoring-grade: plain + * stores + atomic bumps, no lock (events are rare and single-field). */ +struct pcache_cluster_view { + unsigned int generation; /* bumped on every UP/DOWN */ + unsigned int node_ups; /* lifetime UP events */ + unsigned int node_downs; /* lifetime DOWN events */ + unsigned int last_change; /* ticks of the latest event, 0=never */ + int last_node; /* node id of the latest event */ + int last_was_up; /* 1 = UP, 0 = DOWN */ +}; +static struct pcache_cluster_view *pc_view; + +/* What each peer has actually done for us, as opposed to what the clusterer + * says about it. The two can disagree in the way that matters most: the + * membership can read perfectly healthy while the transport carrying pulls + * is dropping every packet, and a bare peer COUNT cannot show that. Keyed + * by node id (1..CL_MAX_NODE_ID); monitoring-grade, so atomic bumps and no + * lock. */ +struct pcache_peer_stat { + unsigned int replies; /* answers of any kind received from it */ + unsigned int values; /* of those, ones that carried a value */ + unsigned int served; /* answers WE sent to it */ + unsigned int last_reply; /* ticks of its last answer, 0 = never */ +}; +static struct pcache_peer_stat *peer_stats; /* [CL_MAX_NODE_ID + 1] */ + +static inline void peer_note_reply(int node_id, int carried_value) +{ + if (!peer_stats || node_id <= 0 || node_id > CL_MAX_NODE_ID) + return; + __sync_fetch_and_add(&peer_stats[node_id].replies, 1); + if (carried_value) + __sync_fetch_and_add(&peer_stats[node_id].values, 1); + peer_stats[node_id].last_reply = get_ticks(); +} + +static inline void peer_note_served(int node_id) +{ + if (!peer_stats || node_id <= 0 || node_id > CL_MAX_NODE_ID) + return; + __sync_fetch_and_add(&peer_stats[node_id].served, 1); +} + +static void pcache_cluster_event(enum clusterer_event ev, int node_id) +{ + if (ev != CLUSTER_NODE_UP && ev != CLUSTER_NODE_DOWN) + return; /* sync-protocol events: we register startup_sync=0 */ + if (!pc_view) + return; + + pc_view->last_node = node_id; + pc_view->last_was_up = (ev == CLUSTER_NODE_UP); + pc_view->last_change = get_ticks(); + if (ev == CLUSTER_NODE_UP) + __sync_fetch_and_add(&pc_view->node_ups, 1); + else + __sync_fetch_and_add(&pc_view->node_downs, 1); + __sync_fetch_and_add(&pc_view->generation, 1); + + LM_INFO("cluster %d membership: node %d went %s (generation %u)\n", + sync_cluster_id, node_id, ev == CLUSTER_NODE_UP ? "UP" : "DOWN", + pc_view->generation); +} + +/* Snapshot the live peer set (the clusterer list holds peers only, not + * this node) plus the membership generation it was taken under. A caller + * that fans work out to these peers re-reads the generation afterwards: + * a change means the set went stale mid-flight. Returns the number of + * ids written, or -1 when cluster sync is not active. + * + * @truncated, when given, says the cluster held more peers than fitted. + * A caller that concludes something from the whole set answering - the + * pull deciding a key is absent - must not draw that conclusion from a + * partial set, because the peers it never counted are exactly the ones + * that might have had it. */ +static int pcache_cluster_members(int *ids, int max, unsigned int *gen, + int *truncated) +{ + clusterer_node_t *list, *n; + int cnt = 0; + + if (truncated) + *truncated = 0; + if (!cluster_ready || !pc_view) + return -1; + if (gen) + *gen = pc_view->generation; + list = clusterer_api.get_nodes(sync_cluster_id); + for (n = list; n; n = n->next) { + if (cnt >= max) { + if (truncated) + *truncated = 1; + break; + } + ids[cnt++] = n->node_id; + } + if (list) + clusterer_api.free_nodes(list); + return cnt; +} +#define PCACHE_SYNC_RELOAD 1 +#define PCACHE_SYNC_VERSION 1 +/* raised on a node that reloaded because a peer issued perf_sync */ +static str evi_synced_name = str_init("E_CACHEDB_PERF_SYNCED"); +static event_id_t evi_synced_id = EVI_ERROR; +static str evp_source_node = str_init("source_node"); +static void pcache_raise_synced(str *coll, int src_node); +/* huge pages requested but the granted tier is sub-optimal; raised once from + * the first maintenance tick, since EVI has no subscribers yet at mod_init. + * The one-shot gate is in shm with an atomic test-and-set, so exactly one + * process raises it however many run the timer */ +static int mem_degraded = 0; +static int *mem_degraded_gate = NULL; + +static int pcache_parse_collections(unsigned int type, void *val); +static int pcache_store_urls(unsigned int type, void *val); +static int w_perf_del(struct sip_msg *msg, str *glob, str *col_s); +static int w_perf_mget(struct sip_msg *msg, str *glob, pv_spec_t *keys_pv, + pv_spec_t *vals_pv, str *col_s, int *limit); +static int w_perf_mget_json(struct sip_msg *msg, str *glob, pv_spec_t *dst_pv, + str *col_s, int *limit); +static int w_perf_sync(struct sip_msg *msg, str *col_s); +static int fixup_check_wvar(void **param); + +/* introspection MI (CP-18) - defined just above the mi_cmds table; these + * forward decls let that table sit before the glob/collection helpers */ +static pcache_col_t *col_by_name(const str *name); +static mi_response_t *mi_perf_cluster_probe_0(const mi_params_t *params, + struct mi_handler *async); +static mi_response_t *mi_perf_cluster_probe_1(const mi_params_t *params, + struct mi_handler *async); +int load_pcache_pull(pcache_pull_api_t *api); +static int pcache_pull_start(pcache_col_t *col, const str *key, + int hint_node, int *fd, unsigned int *id_out); +static int pcache_pull_key(pcache_col_t *col, const str *key, char *out, + unsigned int outlen, unsigned int *vlen, unsigned int *expires); +static int pcache_pull_enabled(pcache_col_t *col); +static char *glob_dup(const str *glob); +static int perf_del_run(pcache_col_t *col, str *glob); +static inline unsigned int ttl_to_abs(int expires); + +#define PERF_ROUTES (REQUEST_ROUTE|ONREPLY_ROUTE|FAILURE_ROUTE|BRANCH_ROUTE|\ + LOCAL_ROUTE|STARTUP_ROUTE|TIMER_ROUTE|EVENT_ROUTE) + +static const cmd_export_t cmds[] = { + {"load_pcache_pull", (cmd_function)load_pcache_pull, {{0,0,0}}, 0}, + {"perf_del", (cmd_function)w_perf_del, { + {CMD_PARAM_STR,0,0}, + {CMD_PARAM_STR|CMD_PARAM_OPT,0,0}, {0,0,0}}, + PERF_ROUTES}, + {"perf_mget", (cmd_function)w_perf_mget, { + {CMD_PARAM_STR,0,0}, + {CMD_PARAM_VAR,fixup_check_wvar,0}, + {CMD_PARAM_VAR,fixup_check_wvar,0}, + {CMD_PARAM_STR|CMD_PARAM_OPT,0,0}, + {CMD_PARAM_INT|CMD_PARAM_OPT,0,0}, {0,0,0}}, + PERF_ROUTES}, + {"perf_mget_json", (cmd_function)w_perf_mget_json, { + {CMD_PARAM_STR,0,0}, + {CMD_PARAM_VAR,fixup_check_wvar,0}, + {CMD_PARAM_STR|CMD_PARAM_OPT,0,0}, + {CMD_PARAM_INT|CMD_PARAM_OPT,0,0}, {0,0,0}}, + PERF_ROUTES}, + {"perf_sync", (cmd_function)w_perf_sync, { + {CMD_PARAM_STR|CMD_PARAM_OPT,0,0}, {0,0,0}}, + PERF_ROUTES}, + {0,0,{{0,0,0}},0} +}; + +static const param_export_t params[] = { + { "cache_collections", STR_PARAM|USE_FUNC_PARAM, + (void *)pcache_parse_collections }, + { "cachedb_url", STR_PARAM|USE_FUNC_PARAM, + (void *)pcache_store_urls }, + { "arena_selftest", INT_PARAM, &arena_selftest }, + { "htable_selftest", INT_PARAM, &htable_selftest }, + { "arena_hugepage_mb", INT_PARAM, &pcache_arena_hugepage_mb }, + { "expiry_sweep_period", INT_PARAM, &expiry_sweep_period }, + { "growth_load_factor", INT_PARAM, &growth_load_factor }, + { "growth_budget", INT_PARAM, &growth_budget }, + { "event_expired_collections", STR_PARAM, &event_expired_collections }, + { "db_url", STR_PARAM, &db_url }, + { "db_table", STR_PARAM, &db_table }, + { "db_mode", INT_PARAM, &db_mode }, + { "persist_collections", STR_PARAM, &persist_collections }, + { "sync_cluster_id", INT_PARAM, &sync_cluster_id }, + { "sync_shtag", STR_PARAM, &sync_shtag_str }, + { "pull_transport", STR_PARAM, &pull_transport_str }, + { "pull_timeout_ms", INT_PARAM, &pull_timeout_ms }, + { "pull_linger_ms", INT_PARAM, &pull_linger_ms }, + { "pull_negative_ms", INT_PARAM, &pull_negative_ms }, + { "pull_on_miss", INT_PARAM, &pull_on_miss }, + { "pull_max_value", INT_PARAM, &pull_max_value }, + { "pull_max_key", INT_PARAM, &pull_max_key }, + { "replicate_collections", STR_PARAM, &replicate_collections }, + {0,0,0} +}; + +/* + * CP-06 statistics: everything is STAT_IS_FUNC - sums of the per-process + * shards computed at read time. No shared counter is ever touched on the + * hot path (DESIGN 2.5 hard rule). + */ +enum pcache_stat_field { + PSF_HITS, PSF_MISSES, PSF_STORES, PSF_REMOVES, PSF_ENTRIES, + PSF_RETRIES, PSF_FALLBACKS, PSF_EXPIRED, PSF_DESTROYED +}; + +static unsigned long pcache_stat_field(enum pcache_stat_field which) +{ + pcache_col_t *col; + pcache_ht_totals_t t; + unsigned long sum = 0; + + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + pcache_ht_totals(col->htable, &t); + switch (which) { + case PSF_HITS: sum += t.hits; break; + case PSF_MISSES: sum += t.misses; break; + case PSF_STORES: sum += t.stores; break; + case PSF_REMOVES: sum += t.removes; break; + case PSF_EXPIRED: sum += t.expired; break; + case PSF_DESTROYED: sum += t.destroyed; break; + case PSF_ENTRIES: sum += t.entries; break; + case PSF_RETRIES: sum += t.retries; break; + case PSF_FALLBACKS: sum += t.fallbacks; break; + } + } + return sum; +} + +#define PSTATF(_fn, _which) \ + static unsigned long _fn(void *ctx) \ + { return pcache_stat_field(_which); } + +PSTATF(smf_hits, PSF_HITS) +PSTATF(smf_misses, PSF_MISSES) +PSTATF(smf_stores, PSF_STORES) +PSTATF(smf_removes, PSF_REMOVES) +PSTATF(smf_expired, PSF_EXPIRED) +PSTATF(smf_destroyed, PSF_DESTROYED) +PSTATF(smf_entries, PSF_ENTRIES) +PSTATF(smf_retries, PSF_RETRIES) +PSTATF(smf_fallbacks, PSF_FALLBACKS) + +/* + * Cross-node pull statistics. + * + * These mirror what perf_stats already reports, but as module statistics so + * Prometheus scrapes them - without that, the only evidence a dashboard has + * that read repair is working is the entry count rising, which shows the + * RESULT and not the mechanism: a node whose every pull times out looks + * exactly like one that simply has no misses. + * + * pull_stats[] is shm and only exists once the pull layer came up, so every + * accessor tolerates it being NULL (pull disabled, or a config that never + * reached that far). + */ +static unsigned long pull_stat(int which) +{ + return pull_stats ? (unsigned long)pull_stats[which] : 0; +} + +#define PULLSTATF(_fn, _which) \ + static unsigned long _fn(void *ctx) { return pull_stat(_which); } + +/* Complain about a pull datagram that never left, at most once every + * PCACHE_PULL_SEND_WARN_IVL seconds. + * + * This used to be LM_DBG, which made it unreachable on every deployed node: + * log_level 3 is INFO and L_DBG is 4. That is the wrong level for it - a send + * that fails is invisible at the far end, so the requester simply times out, + * and this line is the only direct evidence of why. It is the actual cause + * behind a class of "cross-node pull is slow / does not converge" reports. + * + * It cannot be an unconditional LM_WARN either: a partitioned or overloaded + * peer fails every send, and an unbounded warn is its own incident. So warn + * on the first failure, then at most once per interval, carrying the count it + * stands for. The exact total is always available as pulls_send_failed. + * + * Two workers can pass the interval check at once and both warn. That is + * deliberate - it costs an occasional duplicate line and saves taking a lock + * on a failure path. */ +#define PCACHE_PULL_SEND_WARN_IVL 30 + +static void pull_send_failed(const char *what, int dst_node) +{ + unsigned int now = get_ticks(), held; + char tgt[32]; + + if (pull_stats) + __sync_fetch_and_add(&pull_stats[PULL_ST_SEND_FAIL], 1); + if (!pull_send_warn) + return; + + if (pull_send_warn->last != 0 && + now - pull_send_warn->last < PCACHE_PULL_SEND_WARN_IVL) { + __sync_fetch_and_add(&pull_send_warn->suppressed, 1); + return; + } + pull_send_warn->last = now; + held = __sync_lock_test_and_set(&pull_send_warn->suppressed, 0); + + if (dst_node > 0) + snprintf(tgt, sizeof tgt, "node %d", dst_node); + else + snprintf(tgt, sizeof tgt, "the cluster"); + + if (held) + LM_WARN("cross-node pull: %s [%s], and %u more in the last %ds - " + "whoever asked is timing out; see the pulls_send_failed " + "statistic for the running total\n", + what, tgt, held, PCACHE_PULL_SEND_WARN_IVL); + else + LM_WARN("cross-node pull: %s [%s] - whoever asked is timing out\n", + what, tgt); +} + +PULLSTATF(smf_pulls_requested, PULL_ST_REQUESTED) +PULLSTATF(smf_pulls_served, PULL_ST_SERVED) +PULLSTATF(smf_pulls_received, PULL_ST_RECEIVED) +PULLSTATF(smf_pulls_timeout, PULL_ST_TIMEOUT) +PULLSTATF(smf_pulls_stored, PULL_ST_STORED) +PULLSTATF(smf_pulls_suppressed, PULL_ST_SUPPRESSED) +PULLSTATF(smf_pulls_abandoned, PULL_ST_ABANDONED) +PULLSTATF(smf_pulls_send_failed, PULL_ST_SEND_FAIL) +PULLSTATF(smf_pulls_orphaned, PULL_ST_ORPHANED) +PULLSTATF(smf_pulls_late_stored, PULL_ST_LATE_STORED) +PULLSTATF(smf_pulls_orphan_evicted, PULL_ST_ORPHAN_EVICTED) +PULLSTATF(smf_pulls_late_superseded, PULL_ST_LATE_SUPERSEDED) +PULLSTATF(smf_pulls_late_expired, PULL_ST_LATE_EXPIRED) +PULLSTATF(smf_pulls_orphan_expired, PULL_ST_ORPHAN_EXPIRED) +PULLSTATF(smf_pulls_foreign_cluster, PULL_ST_FOREIGN_CLUSTER) +PULLSTATF(smf_pulls_negative, PULL_ST_NEGATIVE) +PULLSTATF(smf_pulls_oversize, PULL_ST_OVERSIZE) +PULLSTATF(smf_pulls_skip_notreplicated, PULL_ST_SKIP_NOTREPLICATED) +PULLSTATF(smf_pulls_skip_toolong, PULL_ST_SKIP_TOOLONG) +PULLSTATF(smf_pulls_skip_nopeers, PULL_ST_SKIP_NOPEERS) +PULLSTATF(smf_pulls_skip_noslot, PULL_ST_SKIP_NOSLOT) + +/* A GAUGE, unlike every other pull stat: it should read 0 whenever nothing is + * being asked. Anything parked here means slots are taken and not released, + * which ends as "all pull slots busy" and silent loss of read repair - so it + * is worth alerting on, where the counters are only worth graphing. */ +static unsigned long smf_pulls_in_flight(void *ctx) +{ + unsigned long busy = 0; + int k; + + if (!pull_slots || !pull_lock) + return 0; + lock_get(pull_lock); + for (k = 0; k < PCACHE_PULL_SLOTS; k++) + /* an orphan is not a pull in flight - nobody is waiting on it. + * This gauge is documented as the one that should sit at 0, so + * counting orphans would fire the leak alarm on the ordinary + * outcome of a timeout. */ + if (pull_slot_at(k)->id && !pull_slot_at(k)->orphan) + busy++; + lock_release(pull_lock); + return busy; +} + +/* Per-collection convergence, registered dynamically in mod_init (one pair per + * declared collection) because the module-wide names above cannot say WHICH + * collection is converging - and with a fetch-only collection like rtpdebug in + * the mix, the aggregate is actively misleading. @ctx is the collection. */ +/* + * "_" in shm, for a dynamically registered statistic. + * + * NOT build_stat_name(): that joins with a HYPHEN, which is fine for the + * per-process pkmem statistics because those are STAT_HIDDEN and only their + * group name is ever exported - but these are meant to be read individually, + * and the prometheus module concatenates a statistic name verbatim with no + * sanitising. A hyphen is not legal in a Prometheus metric name, so + * "default-pulled_from_cluster" would have produced a metric that breaks the + * scrape rather than one that merely looks odd. + */ +static char *pcache_stat_name(pcache_col_t *col, const char *what) +{ + int n = col->col_name.len + 1 + strlen(what) + 1; + char *s = shm_malloc(n); + + if (!s) + return NULL; + snprintf(s, n, "%.*s_%s", col->col_name.len, col->col_name.s, what); + return s; +} + +static unsigned long smf_col_pulled_in(void *ctx) +{ + return ctx ? ((pcache_col_t *)ctx)->pulled_in : 0; +} + +static unsigned long smf_col_served_out(void *ctx) +{ + return ctx ? ((pcache_col_t *)ctx)->served_out : 0; +} + +static unsigned long smf_arena_bytes(void *ctx) +{ + unsigned int c; + unsigned long b; + + pcache_arena_stats(&c, &b); + return b; +} + +static unsigned long smf_arena_chunks(void *ctx) +{ + unsigned int c; + unsigned long b; + + pcache_arena_stats(&c, &b); + return c; +} + +static unsigned long smf_mem_tier_probe(void *ctx) +{ + /* what this host is CAPABLE of - not necessarily what is in use, + * see smf_mem_tier_active() for that */ + return pcache_mem.tier; +} + +static unsigned long smf_mem_tier_active(void *ctx) +{ + /* the tier ACTUALLY backing the dedicated arena_hugepage_mb + * reservation right now; reads as PCACHE_MEM_NO_ARENA (99) whenever + * arena_hugepage_mb is unset/0 or its reservation failed - which is + * also exactly when every cachedb_perf allocation is really going + * through shm_malloc(), so the true page backing is the CORE + * allocator's and is NOT measured here. It used to read 4 (plain 4K), + * which was misread live as "the cache is on small pages" while it sat + * on HG_MALLOC's 2M hugepages. See smf_hugepage_arena_active(). */ + return pcache_arena_tier(); +} + +static unsigned long smf_hugepage_arena_active(void *ctx) +{ + int active; + unsigned long total, used, free; + + pcache_arena_hugepage_capacity(&active, &total, &used, &free); + return active; +} + +static unsigned long smf_hugepage_arena_total_bytes(void *ctx) +{ + int active; + unsigned long total, used, free; + + pcache_arena_hugepage_capacity(&active, &total, &used, &free); + return total; +} + +static unsigned long smf_hugepage_arena_used_bytes(void *ctx) +{ + int active; + unsigned long total, used, free; + + pcache_arena_hugepage_capacity(&active, &total, &used, &free); + return used; +} + +static unsigned long smf_hugepage_arena_free_bytes(void *ctx) +{ + int active; + unsigned long total, used, free; + + pcache_arena_hugepage_capacity(&active, &total, &used, &free); + return free; +} + +static const stat_export_t mod_stats[] = { + {"hits", STAT_IS_FUNC, (stat_var **)smf_hits}, + {"misses", STAT_IS_FUNC, (stat_var **)smf_misses}, + {"stores", STAT_IS_FUNC, (stat_var **)smf_stores}, + {"removes", STAT_IS_FUNC, (stat_var **)smf_removes}, + {"expired", STAT_IS_FUNC, (stat_var **)smf_expired}, + {"destroyed", STAT_IS_FUNC, (stat_var **)smf_destroyed}, + {"entries", STAT_IS_FUNC, (stat_var **)smf_entries}, + {"seqlock_retries", STAT_IS_FUNC, (stat_var **)smf_retries}, + {"lock_fallbacks", STAT_IS_FUNC, (stat_var **)smf_fallbacks}, + {"arena_bytes", STAT_IS_FUNC, (stat_var **)smf_arena_bytes}, + {"arena_chunks", STAT_IS_FUNC, (stat_var **)smf_arena_chunks}, + /* memory_tier renamed to memory_tier_probe (2026-08-07) - the old + * name was mistaken for "what's in use" live during a real + * diagnosis session; not shipped/stable API yet (module unmerged), + * so a rename is safe. See smf_mem_tier_probe()'s comment. */ + {"memory_tier_probe", STAT_IS_FUNC, (stat_var **)smf_mem_tier_probe}, + {"memory_tier_active", STAT_IS_FUNC, (stat_var **)smf_mem_tier_active}, + {"hugepage_arena_active", STAT_IS_FUNC, (stat_var **)smf_hugepage_arena_active}, + {"hugepage_arena_total_bytes", STAT_IS_FUNC, (stat_var **)smf_hugepage_arena_total_bytes}, + {"hugepage_arena_used_bytes", STAT_IS_FUNC, (stat_var **)smf_hugepage_arena_used_bytes}, + {"hugepage_arena_free_bytes", STAT_IS_FUNC, (stat_var **)smf_hugepage_arena_free_bytes}, + /* cross-node pull (CP-15). Module-wide; the per-collection split is + * registered dynamically in mod_init - see smf_col_pulled_in(). */ + {"pulls_requested", STAT_IS_FUNC, (stat_var **)smf_pulls_requested}, + {"pulls_served", STAT_IS_FUNC, (stat_var **)smf_pulls_served}, + {"pulls_received", STAT_IS_FUNC, (stat_var **)smf_pulls_received}, + {"pulls_timed_out", STAT_IS_FUNC, (stat_var **)smf_pulls_timeout}, + {"pulls_stored", STAT_IS_FUNC, (stat_var **)smf_pulls_stored}, + {"pulls_suppressed", STAT_IS_FUNC, (stat_var **)smf_pulls_suppressed}, + {"pulls_abandoned", STAT_IS_FUNC, (stat_var **)smf_pulls_abandoned}, + {"pulls_send_failed", STAT_IS_FUNC, (stat_var **)smf_pulls_send_failed}, + {"pulls_orphaned", STAT_IS_FUNC, (stat_var **)smf_pulls_orphaned}, + {"pulls_late_stored", STAT_IS_FUNC, (stat_var **)smf_pulls_late_stored}, + {"pulls_orphan_evicted", STAT_IS_FUNC, + (stat_var **)smf_pulls_orphan_evicted}, + {"pulls_late_superseded", STAT_IS_FUNC, + (stat_var **)smf_pulls_late_superseded}, + {"pulls_late_expired", STAT_IS_FUNC, + (stat_var **)smf_pulls_late_expired}, + {"pulls_orphan_expired", STAT_IS_FUNC, + (stat_var **)smf_pulls_orphan_expired}, + {"pulls_negative", STAT_IS_FUNC, (stat_var **)smf_pulls_negative}, + {"pulls_oversize", STAT_IS_FUNC, (stat_var **)smf_pulls_oversize}, + {"pulls_in_flight", STAT_IS_FUNC, (stat_var **)smf_pulls_in_flight}, + {"pulls_foreign_cluster", STAT_IS_FUNC, + (stat_var **)smf_pulls_foreign_cluster}, + {"pulls_skip_notreplicated", STAT_IS_FUNC, + (stat_var **)smf_pulls_skip_notreplicated}, + {"pulls_skip_toolong", STAT_IS_FUNC, + (stat_var **)smf_pulls_skip_toolong}, + {"pulls_skip_nopeers", STAT_IS_FUNC, + (stat_var **)smf_pulls_skip_nopeers}, + {"pulls_skip_noslot", STAT_IS_FUNC, + (stat_var **)smf_pulls_skip_noslot}, + {0,0,0} +}; + +/* the perf_stats MI (5.2): per-collection detail the flat stats cannot carry */ +static int mi_stats_fill(mi_item_t *cobj, pcache_col_t *col) +{ + pcache_htable_t *ht = col->htable; + pcache_ht_totals_t t; + unsigned long reads; + const char *note; + double rate; + char buf[32]; + int n; + + pcache_ht_totals(ht, &t); + if (add_mi_string(cobj, MI_SSTR("name"), + col->col_name.s, col->col_name.len) < 0 || + add_mi_number(cobj, MI_SSTR("buckets"), ht->nbuckets) < 0 || + add_mi_number(cobj, MI_SSTR("entries"), t.entries) < 0 || + add_mi_number(cobj, MI_SSTR("overflow"), ht->ovf_count) < 0 || + add_mi_number(cobj, MI_SSTR("hits"), t.hits) < 0 || + add_mi_number(cobj, MI_SSTR("misses"), t.misses) < 0 || + add_mi_number(cobj, MI_SSTR("stores"), t.stores) < 0 || + add_mi_number(cobj, MI_SSTR("removes"), t.removes) < 0 || + add_mi_number(cobj, MI_SSTR("expired"), t.expired) < 0 || + add_mi_number(cobj, MI_SSTR("destroyed"), t.destroyed) < 0 || + add_mi_number(cobj, MI_SSTR("seqlock_retries"), t.retries) < 0 || + add_mi_number(cobj, MI_SSTR("lock_fallbacks"), t.fallbacks) < 0) + return -1; + + reads = t.hits + t.misses; + rate = reads ? 100.0 * t.hits / reads : 0.0; + n = snprintf(buf, sizeof buf, "%.1f", rate); + if (add_mi_string(cobj, MI_SSTR("hit_rate_pct"), buf, n) < 0) + return -1; + /* The counters are cumulative since startup (or the last + * perf_stats_reset), so this is a lifetime average: right after a + * restart it is dragged down by every sequential request whose dialog + * predates the cache, and it recovers only as those age out. Judge a + * running system on the trend between two polls, not on one reading. */ + if (!reads) + note = "no lookups yet"; + else if (!t.stores) + /* Every read has been a miss, but nothing has ever been stored + * either - there is no state to have been "lost" or "expired", + * this collection has simply never been written to (e.g. it + * loaded 0 entries from persistence at startup). A low rate + * here points at nothing reaching this collection at all, not + * at eviction/TTL tuning. */ + note = "no state has ever been stored in this collection - a miss " + "here is not loss or expiry, check whether writes reach " + "this collection and whether persistence loaded any rows"; + else if (rate >= 80.0) + note = "healthy: the large majority of lookups hit"; + else if (rate >= 40.0) + note = "fair: normal while the cache refills after a restart - " + "if it does not climb, state is expiring before it is used"; + else + note = "low: cached state is being lost or is expiring before it " + "is used - expected only shortly after a restart"; + if (add_mi_string(cobj, MI_SSTR("hit_rate_note"), note, strlen(note)) < 0) + return -1; + + /* Cluster sync is on-demand, so report WHEN this node last pushed or + * pulled rather than implying the caches match. -1 = never. Note the + * clusterer's own "Ok" for the cachedb-perf-sync capability only means + * it is registered and enabled - it says nothing about convergence. */ + if (sync_cluster_id > 0) { + if (add_mi_number(cobj, MI_SSTR("pulled_from_cluster"), + col->pulled_in) < 0 || + add_mi_number(cobj, MI_SSTR("served_to_cluster"), + col->served_out) < 0) + return -1; + if (add_mi_number(cobj, MI_SSTR("last_sync_out"), + col->last_sync_out ? + (int)(get_ticks() - col->last_sync_out) : -1) < 0 || + add_mi_number(cobj, MI_SSTR("last_sync_in"), + col->last_sync_in ? + (int)(get_ticks() - col->last_sync_in) : -1) < 0 || + add_mi_number(cobj, MI_SSTR("last_sync_source"), + col->last_sync_src) < 0) + return -1; + } + + n = snprintf(buf, sizeof buf, "%.3f", + (double)t.entries / ht->nbuckets); + if (add_mi_string(cobj, MI_SSTR("load_factor"), buf, n) < 0) + return -1; + reads = t.hits + t.misses; + n = snprintf(buf, sizeof buf, "%.3f", + reads ? 1000.0 * t.retries / reads : 0.0); + return add_mi_string(cobj, MI_SSTR("retries_per_1k_reads"), buf, n); +} + +static mi_response_t *mi_perf_stats(str *col_s) +{ + mi_response_t *resp; + mi_item_t *obj, *arr, *cobj, *aobj, *hobj; + pcache_col_t *col; + const char *tier_probe, *tier_active; + unsigned long bytes, hp_total, hp_used, hp_free; + unsigned int nchunks, matched = 0; + int hp_active; + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + + arr = add_mi_array(obj, MI_SSTR("collections")); + if (!arr) + goto err; + for (col = pcache_collection; col; col = col->next) { + if (col_s && (col->col_name.len != col_s->len || + memcmp(col->col_name.s, col_s->s, col_s->len))) + continue; + if (!col->htable) + continue; + cobj = add_mi_object(arr, NULL, 0); + if (!cobj || mi_stats_fill(cobj, col) < 0) + goto err; + matched++; + } + if (col_s && !matched) { + free_mi_response(resp); + return init_mi_error(404, MI_SSTR("no such collection")); + } + + /* "arena": total cachedb_perf usage regardless of which backing + * actually served it (dedicated reservation OR the shm_malloc + * fallback) - NOT specific to the dedicated arena_hugepage_mb + * reservation, see "hugepage_reservation" below for that. */ + aobj = add_mi_object(obj, MI_SSTR("arena")); + if (!aobj) + goto err; + pcache_arena_stats(&nchunks, &bytes); + if (add_mi_number(aobj, MI_SSTR("chunks"), nchunks) < 0 || + add_mi_number(aobj, MI_SSTR("bytes"), bytes) < 0) + goto err; + + /* _probe = what this host is CAPABLE of (startup capability check, + * see pcache_mem_probe()) - NOT proof anything is actually reserved. + * _active = the tier ACTUALLY backing the dedicated reservation + * right now; reads PCACHE_MEM_4K/"4K" whenever arena_hugepage_mb is + * unset/0 or its reservation failed, which is also exactly when + * every cachedb_perf allocation is really going through plain + * shm_malloc() - counted in core's own shmem: stats, not here. */ + tier_probe = pcache_mem_tier_str(pcache_mem.tier); + tier_active = pcache_mem_tier_str(pcache_arena_tier()); + if (add_mi_number(obj, MI_SSTR("memory_tier_probe"), pcache_mem.tier) < 0 || + add_mi_string(obj, MI_SSTR("memory_backing_probe"), + (char *)tier_probe, strlen(tier_probe)) < 0 || + add_mi_number(obj, MI_SSTR("memory_tier_active"), pcache_arena_tier()) < 0 || + add_mi_string(obj, MI_SSTR("memory_backing_active"), + (char *)tier_active, strlen(tier_active)) < 0) + goto err; + + /* "hugepage_reservation": the DEDICATED arena_hugepage_mb reservation + * specifically - deliberately its OWN object, never folded into + * "arena" above or into core's shmem: stats, so a human or dashboard + * can never double-count or misattribute. "active" MUST be checked + * before trusting the byte counts - all three read 0 whenever no + * dedicated reservation exists, which is NOT the same thing as "a + * reservation exists and is currently empty". */ + hobj = add_mi_object(obj, MI_SSTR("hugepage_reservation")); + if (!hobj) + goto err; + pcache_arena_hugepage_capacity(&hp_active, &hp_total, &hp_used, &hp_free); + if (add_mi_number(hobj, MI_SSTR("active"), hp_active) < 0 || + add_mi_number(hobj, MI_SSTR("total_bytes"), hp_total) < 0 || + add_mi_number(hobj, MI_SSTR("used_bytes"), hp_used) < 0 || + add_mi_number(hobj, MI_SSTR("free_bytes"), hp_free) < 0) + goto err; + + /* Cluster membership, when sync is active. peers_up counts the OTHER + * nodes the clusterer can currently reach; the generation ticks on + * every membership change, so two equal reads bracket a quiet period. */ + if (cluster_ready && pc_view) { + mi_item_t *clobj = add_mi_object(obj, MI_SSTR("cluster")); + int ids[CL_MAX_NODE_ID], nup; + unsigned int gen = 0; + + if (!clobj) + goto err; + nup = pcache_cluster_members(ids, CL_MAX_NODE_ID, &gen, NULL); + if (add_mi_number(clobj, MI_SSTR("cluster_id"), sync_cluster_id) < 0 || + /* which of the peers below is us - the list holds peers only */ + add_mi_number(clobj, MI_SSTR("my_node_id"), + clusterer_api.get_my_id ? clusterer_api.get_my_id() : 0) < 0 || + add_mi_number(clobj, MI_SSTR("peers_up"), nup < 0 ? 0 : nup) < 0 || + add_mi_number(clobj, MI_SSTR("membership_generation"), gen) < 0 || + add_mi_number(clobj, MI_SSTR("node_ups"), pc_view->node_ups) < 0 || + add_mi_number(clobj, MI_SSTR("node_downs"), + pc_view->node_downs) < 0 || + add_mi_number(clobj, MI_SSTR("last_change_ago"), + pc_view->last_change ? + (int)(get_ticks() - pc_view->last_change) : -1) < 0) + goto err; + if (pull_ready && pull_stats && + (add_mi_number(clobj, MI_SSTR("pulls_requested"), + pull_stats[PULL_ST_REQUESTED]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_received"), + pull_stats[PULL_ST_RECEIVED]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_served"), + pull_stats[PULL_ST_SERVED]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_timed_out"), + pull_stats[PULL_ST_TIMEOUT]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_stored"), + pull_stats[PULL_ST_STORED]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_suppressed"), + pull_stats[PULL_ST_SUPPRESSED]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_send_failed"), + pull_stats[PULL_ST_SEND_FAIL]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_negative"), + pull_stats[PULL_ST_NEGATIVE]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_oversize"), + pull_stats[PULL_ST_OVERSIZE]) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_skip_notreplicated"), + pull_stat(PULL_ST_SKIP_NOTREPLICATED)) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_skip_toolong"), + pull_stat(PULL_ST_SKIP_TOOLONG)) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_skip_nopeers"), + pull_stat(PULL_ST_SKIP_NOPEERS)) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_skip_noslot"), + pull_stat(PULL_ST_SKIP_NOSLOT)) < 0 || + add_mi_number(clobj, MI_SSTR("pulls_abandoned"), + pull_stats[PULL_ST_ABANDONED]) < 0)) + goto err; + /* in-flight requests: a gauge, not a counter. It should sit at 0 + * when nothing is being asked; anything else parked there means + * slots are being taken and not released, which ends as "all pull + * slots busy" and silent loss of read repair. */ + if (pull_ready && pull_slots) { + int busy = 0, k; + + lock_get(pull_lock); + for (k = 0; k < PCACHE_PULL_SLOTS; k++) + if (pull_slot_at(k)->id) + busy++; + lock_release(pull_lock); + if (add_mi_number(clobj, MI_SSTR("pulls_in_flight"), busy) < 0 || + add_mi_number(clobj, MI_SSTR("pull_slots"), + PCACHE_PULL_SLOTS) < 0) + goto err; + } + + /* Who the peers are, and whether they are actually answering US. + * A peer count alone cannot tell a healthy cluster apart from one + * whose membership is fine while the transport carrying pulls is + * black-holing every packet - the two look identical until you + * see that no peer has ever replied. So each peer is listed with + * both views side by side: `membership` is what the clusterer + * believes, `replies`/`last_reply_ago` are what this node has + * actually received from it. */ + { + mi_item_t *parr = add_mi_array(clobj, MI_SSTR("topology")); + clusterer_node_t *list, *n; + unsigned int now = get_ticks(); + int me = clusterer_api.get_my_id ? clusterer_api.get_my_id() : 0; + mi_item_t *self; + + if (!parr) + goto err; + + /* this node first - the clusterer list holds peers only, so a + * topology built from it alone silently omits the one node the + * reader is talking to */ + self = add_mi_object(parr, NULL, 0); + if (!self || + add_mi_number(self, MI_SSTR("node_id"), me) < 0 || + add_mi_string(self, MI_SSTR("role"), MI_SSTR("self")) < 0 || + add_mi_string(self, MI_SSTR("membership"), + MI_SSTR("up")) < 0) + goto err; + /* Which address this node actually uses on the cluster plane, + * and which of the three resolution paths produced it. Node + * ids are assigned by the controller and do not follow the + * hosts' addresses in any readable order, so without this a + * reader cannot tell which box they are looking at. It is + * also the fastest way to spot the failure that matters: + * a node resolving its own IP onto the wrong interface. */ +#ifdef CLUSTERER_CTRL_SUPPORT + if (pull_via_clctr && clctr_api.get_my_ip) { + const char *mip = NULL, *mif = NULL, *msrc = NULL; + + if (clctr_api.get_my_ip(&mip, &mif, &msrc) == 0 && mip) { + if (add_mi_string(self, MI_SSTR("ip"), + (char *)mip, strlen(mip)) < 0 || + (mif && add_mi_string(self, MI_SSTR("interface"), + (char *)mif, strlen(mif)) < 0) || + (msrc && add_mi_string(self, MI_SSTR("ip_source"), + (char *)msrc, strlen(msrc)) < 0)) + goto err; + } + } +#endif + + list = cluster_ready ? + clusterer_api.get_nodes(sync_cluster_id) : NULL; + for (n = list; n; n = n->next) { + mi_item_t *p = add_mi_object(parr, NULL, 0); + struct pcache_peer_stat *ps = peer_stats && n->node_id > 0 && + n->node_id <= CL_MAX_NODE_ID ? + &peer_stats[n->node_id] : NULL; + const char *verdict; + int ago; + + if (!p) + goto err; + if (add_mi_number(p, MI_SSTR("node_id"), n->node_id) < 0 || + add_mi_string(p, MI_SSTR("role"), MI_SSTR("peer")) < 0 || + /* what the clusterer believes about it */ + add_mi_string(p, MI_SSTR("membership"), + MI_SSTR("up")) < 0) + goto err; + if (n->description.s && n->description.len && + add_mi_string(p, MI_SSTR("description"), + n->description.s, n->description.len) < 0) + goto err; + if (n->sip_addr.s && n->sip_addr.len && + add_mi_string(p, MI_SSTR("sip_addr"), + n->sip_addr.s, n->sip_addr.len) < 0) + goto err; + /* the peer's address on the cluster plane - see the note on + * the self entry above; node_id alone does not identify a + * host to a human reading these stats */ + { + struct ip_addr pip; + char *pips; + + su2ip_addr(&pip, &n->addr); + pips = ip_addr2a(&pip); + if (pips && *pips && + add_mi_string(p, MI_SSTR("ip"), pips, + strlen(pips)) < 0) + goto err; + } + + if (!ps) + continue; + /* ...and what it has actually done for us. These two can + * disagree in the way that matters: a membership can read + * perfectly healthy while the transport carrying pulls + * drops every packet, and only this half shows it. */ + ago = ps->last_reply ? (int)(now - ps->last_reply) : -1; + verdict = !ps->replies ? "never-answered" + : (ago <= PCACHE_PEER_FRESH_S ? "answering" + : "quiet"); + if (add_mi_string(p, MI_SSTR("pull_health"), + verdict, strlen(verdict)) < 0 || + add_mi_number(p, MI_SSTR("replies"), ps->replies) < 0 || + add_mi_number(p, MI_SSTR("replies_with_value"), + ps->values) < 0 || + add_mi_number(p, MI_SSTR("answers_we_sent_it"), + ps->served) < 0 || + add_mi_number(p, MI_SSTR("last_reply_ago"), ago) < 0) + goto err; + } + if (list) + clusterer_api.free_nodes(list); + } + if (pc_view->last_change) { + char lbuf[32]; + int ln = snprintf(lbuf, sizeof lbuf, "%s:%d", + pc_view->last_was_up ? "up" : "down", pc_view->last_node); + if (add_mi_string(clobj, MI_SSTR("last_event"), lbuf, ln) < 0) + goto err; + } + } + + return resp; +err: + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("Internal error")); +} + +static mi_response_t *mi_perf_stats_1(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + return mi_perf_stats(NULL); +} + +static mi_response_t *mi_perf_stats_2(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + str c; + + if (get_mi_string_param(params, "collection", &c.s, &c.len) < 0) + return init_mi_param_error(); + return mi_perf_stats(&c); +} + +/* + * perf_stats_reset - re-baseline the cumulative counters. + * + * hits/misses/stores/removes/expired/destroyed/retries are running totals + * since startup, so the rates derived from them are lifetime averages: a + * burst of misses right after a restart keeps dragging the hit rate down + * long after the cache has recovered. Resetting gives a clean interval to + * measure over without restarting OpenSIPS. + * + * The counters themselves are not rewound - the hot paths own their per + * process cache lines and must never be written from another process. Only + * a baseline is recorded, and pcache_ht_totals() reports the difference. + * Live gauges (entries, buckets, overflow, load factor, arena) are derived + * from current state, not from the counters, so a reset does not disturb them. + */ +static mi_response_t *mi_perf_stats_reset(str *col_s) +{ + mi_response_t *resp; + mi_item_t *obj; + pcache_col_t *col; + unsigned int matched = 0; + + for (col = pcache_collection; col; col = col->next) { + if (col_s && (col->col_name.len != col_s->len || + memcmp(col->col_name.s, col_s->s, col_s->len))) + continue; + if (!col->htable) + continue; + pcache_ht_stats_reset(col->htable); + matched++; + } + if (col_s && !matched) + return init_mi_error(404, MI_SSTR("no such collection")); + + resp = init_mi_result_object(&obj); + if (!resp) + return init_mi_error(500, MI_SSTR("Internal error")); + if (add_mi_number(obj, MI_SSTR("collections_reset"), matched) < 0) { + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("Internal error")); + } + return resp; +} + +static mi_response_t *mi_perf_stats_reset_1(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + return mi_perf_stats_reset(NULL); +} + +static mi_response_t *mi_perf_stats_reset_2(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + str c; + + if (get_mi_string_param(params, "collection", &c.s, &c.len) < 0) + return init_mi_param_error(); + return mi_perf_stats_reset(&c); +} + +/* + * Introspection MI (CP-18, DESIGN 5.2). Every command is perf_-prefixed to + * match the script functions and to stay clear of the core's bare get/set. + * The walkers are lock-free (seqlock reads), so unlike cachedb_local's scan + * they never stall writers; keys/dump are bounded and scan is the cursored + * answer for anything large. + */ +#define PCACHE_MI_DEF_LIMIT 1000 + +struct mi_walk_ctx { + const char *pat; /* fnmatch pattern, NULL = match all */ + mi_item_t *arr; + unsigned int limit; /* 0 = unbounded (scan bounds by buckets) */ + unsigned int now; + int with_values; /* dump vs keys */ + unsigned int n; + int err; +}; + +static int mi_walk_cb(const str *key, const str *val, unsigned int exp, void *p) +{ + struct mi_walk_ctx *w = p; + mi_item_t *o; + int ttl; + + if (exp && exp <= w->now) + return 0; /* expired-as-absent (3.5) */ + if (w->pat && fnmatch(w->pat, key->s, 0)) + return 0; + + o = add_mi_object(w->arr, NULL, 0); + if (!o) + goto oom; + if (add_mi_string(o, MI_SSTR("key"), (char *)key->s, key->len) < 0) + goto oom; + ttl = exp ? (int)(exp - w->now) : -1; /* -1 = never expires */ + if (add_mi_number(o, MI_SSTR("ttl"), ttl) < 0) + goto oom; + if (w->with_values && + add_mi_string(o, MI_SSTR("value"), (char *)val->s, val->len) < 0) + goto oom; + + w->n++; + if (w->limit && w->n >= w->limit) + return -1; /* stop: limit reached */ + return 0; +oom: + w->err = 1; + return -1; +} + +/* backs perf_keys (with_values = 0) and perf_dump (with_values = 1) */ +static mi_response_t *do_perf_keys(str *glob, str *col_s, int limit, + int with_values) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj, *arr; + struct mi_walk_ctx w; + char *pat = NULL; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + if (glob && glob->len) { + pat = glob_dup(glob); + if (!pat) + return init_mi_error(500, MI_SSTR("out of memory")); + } + + resp = init_mi_result_object(&obj); + if (!resp) { + if (pat) + pkg_free(pat); + return NULL; + } + arr = add_mi_array(obj, MI_SSTR("keys")); + if (!arr) + goto err; + + memset(&w, 0, sizeof w); + w.pat = pat; + w.arr = arr; + w.limit = limit > 0 ? (unsigned int)limit : PCACHE_MI_DEF_LIMIT; + w.now = get_ticks(); + w.with_values = with_values; + pcache_ht_iter(col->htable, mi_walk_cb, &w); + if (pat) { + pkg_free(pat); + pat = NULL; + } + if (w.err) + goto err; + + if (add_mi_number(obj, MI_SSTR("returned"), w.n) < 0) + goto err; + /* tell the operator the result was cut so they narrow it or use scan */ + if (w.n >= w.limit && add_mi_string(obj, MI_SSTR("note"), + MI_SSTR("limit reached - truncated; narrow the glob or use perf_scan")) < 0) + goto err; + return resp; +err: + if (pat) + pkg_free(pat); + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); +} + +/* perf_scan [glob] [count] - cursored, on the default collection */ +static mi_response_t *do_perf_scan(int cursor_in, str *glob, int count) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj, *arr; + struct mi_walk_ctx w; + char *pat = NULL; + unsigned int cur; + + if (cursor_in < 0) + return init_mi_param_error(); + col = col_by_name(NULL); /* the groupless default collection */ + if (!col) + return init_mi_error(404, MI_SSTR("no default collection")); + if (glob && glob->len) { + pat = glob_dup(glob); + if (!pat) + return init_mi_error(500, MI_SSTR("out of memory")); + } + + resp = init_mi_result_object(&obj); + if (!resp) { + if (pat) + pkg_free(pat); + return NULL; + } + arr = add_mi_array(obj, MI_SSTR("keys")); + if (!arr) + goto err; + + memset(&w, 0, sizeof w); + w.pat = pat; + w.arr = arr; + w.limit = 0; /* bucket-bounded: no per-entry stop */ + w.now = get_ticks(); + w.with_values = 0; /* SCAN returns names + ttl */ + cur = (unsigned int)cursor_in; + pcache_ht_scan(col->htable, &cur, count > 0 ? (unsigned int)count : 0, + mi_walk_cb, &w); + if (pat) { + pkg_free(pat); + pat = NULL; + } + if (w.err) + goto err; + + /* cursor 0 = iteration complete; feed any other value back verbatim */ + if (add_mi_number(obj, MI_SSTR("cursor"), cur) < 0 || + add_mi_number(obj, MI_SSTR("returned"), w.n) < 0) + goto err; + return resp; +err: + if (pat) + pkg_free(pat); + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); +} + +/* perf_get [collection] - value + TTL + size for one key */ +/* perf_pull [collection] - ask the cluster for a key this node does + * not have. The MI face exists to exercise and observe the protocol on + * its own, before a SIP path uses it: it reports where the answer came + * from, which is what makes a failing pull diagnosable. */ +static mi_response_t *do_perf_pull(str *key, str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + char buf[PCACHE_PULL_MAX_VAL]; + unsigned int vlen = 0, exp = 0; + int rc; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + if (!pull_ready) + return init_mi_error(500, + MI_SSTR("cross-node pull not active (replicate_collections)")); + if (!col->replicate) + return init_mi_error(500, + MI_SSTR("collection is not in replicate_collections")); + + /* a local hit needs no cluster at all - say so plainly */ + if (pcache_ht_probe(col->htable, key, &vlen, &exp, NULL) == 0) { + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_string(obj, MI_SSTR("source"), MI_SSTR("local")) < 0 || + add_mi_number(obj, MI_SSTR("size"), vlen) < 0 || + add_mi_number(obj, MI_SSTR("ttl"), + exp ? (int)(exp - get_ticks()) : -1) < 0) + goto err; + return resp; + } + + rc = pcache_pull_key(col, key, buf, sizeof buf, &vlen, &exp); + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_string(obj, MI_SSTR("source"), + rc == 1 ? "cluster" : (rc == 0 ? "absent" : "no-answer"), + rc == 1 ? 7 : (rc == 0 ? 6 : 9)) < 0) + goto err; + if (rc == 1) { + if (add_mi_string(obj, MI_SSTR("value"), buf, vlen) < 0 || + add_mi_number(obj, MI_SSTR("size"), vlen) < 0 || + add_mi_number(obj, MI_SSTR("ttl"), + exp ? (int)(exp - get_ticks()) : -1) < 0) + goto err; + } + return resp; +err: + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); +} + +/* perf_probe [collection] - is the key here, and what does it look + * like? Deliberately never returns the value: this is the existence test + * a cross-node lookup would run on a peer, so it must cost what that costs + * (no allocation, no copy, the payload never touched). */ +static mi_response_t *do_perf_probe(str *key, str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + unsigned int vlen = 0, exp = 0; + int rc; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + + rc = pcache_ht_probe(col->htable, key, &vlen, &exp, NULL); + if (rc == -2) + return init_mi_error(404, MI_SSTR("key not found")); + if (rc < 0) + return init_mi_error(500, MI_SSTR("internal error")); + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_string(obj, MI_SSTR("key"), key->s, key->len) < 0 || + add_mi_number(obj, MI_SSTR("size"), vlen) < 0 || + add_mi_number(obj, MI_SSTR("ttl"), + exp ? (int)(exp - get_ticks()) : -1) < 0) { + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); + } + return resp; +} + +static mi_response_t *do_perf_get(str *key, str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + str val; + unsigned int exp = 0, now; + int rc, ttl; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + + rc = pcache_ht_fetch_ex(col->htable, key, &val, &exp); + if (rc == -2) + return init_mi_error(404, MI_SSTR("key not found")); + if (rc < 0) + return init_mi_error(500, MI_SSTR("internal error")); + + now = get_ticks(); + ttl = exp ? (int)(exp - now) : -1; + + resp = init_mi_result_object(&obj); + if (!resp) { + pkg_free(val.s); + return NULL; + } + if (add_mi_string(obj, MI_SSTR("key"), key->s, key->len) < 0 || + add_mi_string(obj, MI_SSTR("value"), val.s, val.len) < 0 || + add_mi_number(obj, MI_SSTR("size"), val.len) < 0 || + add_mi_number(obj, MI_SSTR("ttl"), ttl) < 0) { + pkg_free(val.s); + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); + } + pkg_free(val.s); + return resp; +} + +/* perf_set [ttl] [collection] - single key write */ +static mi_response_t *do_perf_set(str *key, str *value, int ttl, str *col_s) +{ + pcache_col_t *col; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + if (pcache_ht_store(col->htable, key, value, ttl_to_abs(ttl)) < 0) + return init_mi_error(500, MI_SSTR("store failed")); + return init_mi_result_ok(); +} + +/* perf_del [collection] - the MI face of the perf_del() script fn */ +static mi_response_t *do_perf_del_mi(str *glob, str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + int removed; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + removed = perf_del_run(col, glob); + if (removed < 0) + return init_mi_error(500, MI_SSTR("out of memory - deletion partial")); + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_number(obj, MI_SSTR("deleted"), removed) < 0) { + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); + } + return resp; +} + +/* re-arm the TTL of every live key matching a glob (perf_ttl): collect the + * matches lock-free, then touch each - like perf_del, not an atomic snapshot */ +struct touch_ctx { + const char *pat; + str *keys; + unsigned int n, cap, now; + int oom; +}; + +static int touch_collect_cb(const str *key, const str *val, unsigned int exp, + void *p) +{ + struct touch_ctx *tc = p; + str *grown; + + if (exp && exp <= tc->now) /* skip expired: never revive them */ + return 0; + if (fnmatch(tc->pat, key->s, 0)) + return 0; + if (tc->n == tc->cap) { + tc->cap = tc->cap ? 2 * tc->cap : 64; + grown = pkg_realloc(tc->keys, tc->cap * sizeof *tc->keys); + if (!grown) { + tc->oom = 1; + return -1; + } + tc->keys = grown; + } + if (pkg_str_dup(&tc->keys[tc->n], key) < 0) { + tc->oom = 1; + return -1; + } + tc->n++; + return 0; +} + +static int perf_touch_run(pcache_col_t *col, str *glob, unsigned int expires) +{ + struct touch_ctx tc; + char *pat; + unsigned int i, touched = 0; + + pat = glob_dup(glob); + if (!pat) + return -1; + memset(&tc, 0, sizeof tc); + tc.pat = pat; + tc.now = get_ticks(); + pcache_ht_iter(col->htable, touch_collect_cb, &tc); + + for (i = 0; i < tc.n; i++) { + if (pcache_ht_touch(col->htable, &tc.keys[i], expires) == 1) + touched++; + pkg_free(tc.keys[i].s); + } + if (tc.keys) + pkg_free(tc.keys); + pkg_free(pat); + if (tc.oom) { + LM_ERR("out of pkg memory mid-walk - re-arm is partial\n"); + return -1; + } + return (int)touched; +} + +/* perf_ttl [collection] - re-arm the TTL of matching keys */ +static mi_response_t *do_perf_ttl(str *glob, int ttl, str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + int touched; + + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + touched = perf_touch_run(col, glob, ttl_to_abs(ttl)); + if (touched < 0) + return init_mi_error(500, MI_SSTR("out of memory - update partial")); + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_number(obj, MI_SSTR("updated"), touched) < 0) { + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); + } + return resp; +} + +/* perf_save / perf_load [collection] - persist to / restore from the DB + * backend; with no collection, all declared collections */ +static mi_response_t *do_perf_persist(str *col_s, int save) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + int total = 0, ncol = 0, rc; + + if (!pcache_db_enabled()) + return init_mi_error(500, + MI_SSTR("no DB backend configured (set db_url)")); + + if (col_s) { + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + rc = save ? pcache_db_save(col) : pcache_db_load(col); + if (rc < 0) + return init_mi_error(500, MI_SSTR("DB operation failed")); + total = rc; + ncol = 1; + } else { + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + rc = save ? pcache_db_save(col) : pcache_db_load(col); + if (rc < 0) + return init_mi_error(500, MI_SSTR("DB operation failed")); + total += rc; + ncol++; + } + } + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_number(obj, MI_SSTR("collections"), ncol) < 0) + goto err; + if (save) { + if (add_mi_number(obj, MI_SSTR("saved"), total) < 0) + goto err; + } else { + if (add_mi_number(obj, MI_SSTR("loaded"), total) < 0) + goto err; + } + return resp; +err: + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); +} + +/* + * CP-19 Stage 2: cluster sync. The DB is the shared source of truth; a sync + * is save-then-broadcast (the issuing node writes its collection to the DB, + * then signals peers to reload it) - never per-operation replication, and no + * locking. perf_sync OVERWRITES a peer's copy from the DB, so it is meant + * for single-writer / read-replica topologies; a node that also takes local + * writes would lose the unsaved ones. With no clusterer (or cluster_id 0) it + * degrades to a DB save only. + */ + +/* a peer signalled "reload collection X": pull it from the DB and announce */ +/* ===================================================================== + * CP-15.5: pull a key from the cluster on a local miss (read repair) + * + * A node that misses asks the cluster for that one key and uses the + * answer. Pull rather than eager push because the request is issued at + * the moment of need, so it cannot race the traffic the way a broadcast + * on write does - and because misses are the only thing that pays. + * + * Every peer answers, positively or negatively (R5): with a handful of + * nodes the extra packets are trivial and definitive absence is worth + * far more than saving them, because "nobody has it" is then a fact + * rather than a timeout. The probe of CP-15.2 is what makes answering + * cheap - a negative costs the bucket's tag word and nothing else. + * ===================================================================== */ + +static unsigned int neg_hash(pcache_col_t *col, const str *key) +{ + unsigned int h = core_hash((str *)key, &col->col_name, 0); + + return h ? h : 1; /* 0 marks a free slot */ +} + +/* Did we recently establish that nobody has this key? */ +static int pcache_neg_check(pcache_col_t *col, const str *key) +{ + struct pcache_neg_slot *sl; + unsigned int h; + int hit = 0; + + if (!neg_slots || pull_negative_ms <= 0) + return 0; + h = neg_hash(col, key); + sl = &neg_slots[h % PCACHE_NEG_SLOTS]; + + lock_get(neg_lock); + if (sl->hash == h && sl->klen == key->len && + !memcmp(sl->key, key->s, key->len) && + sl->collen == col->col_name.len && + !memcmp(sl->col, col->col_name.s, sl->collen)) { + if (sl->deadline > get_uticks()) + hit = 1; + else + sl->hash = 0; /* lapsed - ask again */ + } + lock_release(neg_lock); + return hit; +} + +static void pcache_neg_add(pcache_col_t *col, const str *key) +{ + struct pcache_neg_slot *sl; + unsigned int h; + + if (!neg_slots || pull_negative_ms <= 0 || + key->len > pull_max_key || col->col_name.len > 63) + return; + h = neg_hash(col, key); + sl = &neg_slots[h % PCACHE_NEG_SLOTS]; + + lock_get(neg_lock); + sl->hash = h; + sl->deadline = get_uticks() + (utime_t)pull_negative_ms * 1000; + sl->klen = key->len; + memcpy(sl->key, key->s, key->len); + sl->collen = col->col_name.len; + memcpy(sl->col, col->col_name.s, sl->collen); + lock_release(neg_lock); +} + +/* A local write makes the key exist here, so whatever we concluded about + * the cluster no longer describes it. */ +static void pcache_neg_clear(pcache_col_t *col, const str *key) +{ + struct pcache_neg_slot *sl; + unsigned int h; + + if (!neg_slots || pull_negative_ms <= 0) + return; + h = neg_hash(col, key); + sl = &neg_slots[h % PCACHE_NEG_SLOTS]; + if (sl->hash != h) + return; /* cheap check before the lock */ + + lock_get(neg_lock); + if (sl->hash == h && sl->klen == key->len && + !memcmp(sl->key, key->s, key->len)) + sl->hash = 0; + lock_release(neg_lock); +} + +/* Is this collection opted in? Nothing pulls unless an operator said so: + * a pull only makes sense where keys are globally meaningful, which the + * module cannot know and must not assume (R2). */ +static int pcache_pull_enabled(pcache_col_t *col) +{ + return pull_ready && col && col->replicate; +} + +static struct pcache_pull_slot *pull_slot_get(unsigned int id) +{ + int i; + + for (i = 0; i < PCACHE_PULL_SLOTS; i++) + if (pull_slot_at(i)->id == id) + return pull_slot_at(i); + return NULL; +} + +/* Send one reply, over the transport the request came in on. + * + * @via_clctr says how it arrived, and is deliberately NOT this node's own + * configuration: the two can differ while a cluster is being reconfigured, + * and answering a BIN request over the multicast plane (or the reverse) + * means the requester waits out its timeout for an answer that was sent, + * which is indistinguishable from packet loss. */ +static void pcache_pull_send_rpl(int dst_node, unsigned int id, const str *key, + int found, int ttl, const str *val, int via_clctr) +{ +#ifdef CLUSTERER_CTRL_SUPPORT + if (via_clctr) { + char buf[CLCTR_MAX_PAYLOAD]; + str pl; + uint32_t id_be = htonl(id), ttl_be = htonl((uint32_t)ttl); + uint16_t kl = htons((uint16_t)key->len); + int vlen = (found == PCACHE_FOUND_YES) ? val->len : 0; + uint16_t vl = htons((uint16_t)vlen); + int n = 0; + + /* Never trust the caller to have sized this: the key is echoed + * straight back from the request, so a peer sending an oversized + * one would otherwise write past the buffer. The serve path + * rejects those already - this is the second lock on the door. */ + if (PCACHE_CLCTR_RPL_HDR + key->len + vlen > (int)sizeof buf) { + LM_ERR("pull reply for a %d byte key with %d bytes of value " + "does not fit %d - dropping it\n", key->len, vlen, + (int)sizeof buf); + return; + } + + buf[n++] = PCACHE_CLCTR_RPL; + memcpy(buf + n, &id_be, 4); n += 4; + buf[n++] = (char)found; + memcpy(buf + n, &ttl_be, 4); n += 4; + memcpy(buf + n, &kl, 2); n += 2; + memcpy(buf + n, key->s, key->len); n += key->len; + memcpy(buf + n, &vl, 2); n += 2; + if (found == PCACHE_FOUND_YES) { + memcpy(buf + n, val->s, val->len); + n += val->len; + } + pl.s = buf; + pl.len = n; + if (clctr_api.send_ucast(sync_cluster_id, dst_node, &pull_channel, + &pl, 0) < 0) + pull_send_failed("a reply did not get through", dst_node); + else if (found == PCACHE_FOUND_YES) + __sync_fetch_and_add(&pull_stats[PULL_ST_SERVED], 1); + return; + } +#endif + + { + bin_packet_t out; + str empty = {NULL, 0}; + + if (bin_init(&out, &pcache_sync_cap, PCACHE_PULL_RPL, + PCACHE_SYNC_VERSION, 0) < 0) + return; + if (bin_push_int(&out, (int)id) < 0 || + bin_push_str(&out, (str *)key) < 0 || + bin_push_int(&out, found) < 0 || + bin_push_int(&out, ttl) < 0 || + bin_push_str(&out, found == PCACHE_FOUND_YES ? (str *)val + : &empty) < 0) { + bin_free_packet(&out); + return; + } + if (clusterer_api.send_to(&out, sync_cluster_id, dst_node) != + CLUSTERER_SEND_SUCCESS) + pull_send_failed("a reply did not get through", dst_node); + else if (found == PCACHE_FOUND_YES) + __sync_fetch_and_add(&pull_stats[PULL_ST_SERVED], 1); + bin_free_packet(&out); + } +} + +/* Answer a peer's request for one key. Transport-neutral: both the BIN + * and the controller-plane receivers decode their own framing and land + * here, so the two can never disagree about what is served. */ +static void pcache_pull_do_serve(int src_node, unsigned int id, str *coll, + str *key, int via_clctr) +{ + pcache_col_t *col; + str val = {NULL, 0}; + unsigned int exp = 0; + int found = PCACHE_FOUND_NO, ttl_left = 0, budget; + + /* The key arrives from a peer and is echoed back in the reply, so it + * is sized before anything else touches it. A requester never asks + * for more than pull_max_key; anything longer is a peer that + * is broken, of another version, or hostile, and answering it at all + * would mean copying it into a fixed reply buffer. */ + if (key->len <= 0 || key->len > pull_max_key || + coll->len <= 0 || coll->len > 63) { + LM_ERR("pull request from node %d has a %d byte key in a %d byte " + "collection - out of range, ignored\n", src_node, key->len, + coll->len); + return; + } + + col = col_by_name(coll); + if (!col || !col->htable || !col->replicate) + goto reply; + + { + int is_counter = 0; + + /* Classify before reading: a native counter counts what happened + * on THIS node, so handing it to a peer would import our tally as + * if it were theirs - and the read path formats it as a decimal + * string, which would silently arrive as a plain value and stop + * being a counter at all. Refuse to serve one; the requester + * treats it as "not here", which is the truth from its side. */ + if (pcache_ht_probe(col->htable, key, NULL, NULL, &is_counter) == 0 + && is_counter) { + LM_DBG("pull: <%.*s> is a counter - not portable, not served\n", + key->len, key->s); + goto reply; + } + } + + if (pcache_ht_fetch_ex(col->htable, key, &val, &exp) != 0) + goto reply; + + /* Hand over the ORIGINAL lifetime, never a fresh TTL: a copy that + * outlives the owner's entry would serve state the owner already + * dropped (R6). 0 = never expires. */ + if (exp) { + unsigned int now = get_ticks(); + + if (exp <= now) { /* raced the sweep - treat as absent */ + pkg_free(val.s); + val.s = NULL; + goto reply; + } + ttl_left = (int)(exp - now); + } + + /* The controller plane is one datagram, so a large value cannot ride + * it. Say "I have it but cannot send it" rather than "not here": + * the requester must not conclude the key is absent from a node that + * demonstrably holds it. */ +#ifdef CLUSTERER_CTRL_SUPPORT + budget = via_clctr + ? CLCTR_MAX_PAYLOAD - (int)(PCACHE_CLCTR_RPL_HDR + key->len) + : pull_max_value; +#else + budget = pull_max_value; +#endif + if (val.len > budget) { + LM_DBG("pull: <%.*s> is %d bytes, over this transport's %d - " + "reporting held-but-unsendable\n", key->len, key->s, val.len, + budget); + found = PCACHE_FOUND_OVERSIZE; + } else { + found = PCACHE_FOUND_YES; + /* counted here, not in pcache_pull_send_rpl(): that helper is + * transport framing and has no collection in scope. Only a real + * value counts - a "not here"/oversize answer is not a serve. */ + if (col) + __sync_fetch_and_add(&col->served_out, 1); + } + +reply: + peer_note_served(src_node); + pcache_pull_send_rpl(src_node, id, key, found, ttl_left, &val, via_clctr); + if (val.s) + pkg_free(val.s); +} + +/* BIN framing -> the shared serve path */ +static void pcache_pull_serve(bin_packet_t *in) +{ + str coll, key; + unsigned int id; + + if (bin_pop_int(in, (int *)&id) < 0 || bin_pop_str(in, &coll) < 0 || + bin_pop_str(in, &key) < 0) { + LM_ERR("malformed pull request from node %d\n", in->src_id); + return; + } + /* arrived over BIN, so it is answered over BIN - even on a node whose + * own pull_transport is the controller plane */ + pcache_pull_do_serve(in->src_id, id, &coll, &key, 0); +} + +/* A peer answered. Fill the waiting slot; first positive answer wins and + * later ones are dropped (several nodes may hold the key once pulls have + * converged). Transport-neutral, like the serve path. */ +static void pcache_pull_do_reply(int src_node, unsigned int id, str *key, + int found, int ttl_left, str *val) +{ + struct pcache_pull_slot *sl; + /* copied out under the lock, used once it is released */ + char late_key[PCACHE_PULL_MAX_KEY], late_col[64]; + char late_val[PCACHE_PULL_MAX_VAL]; + unsigned int late_len = 0, late_exp = 0; + int late_kl = 0, late_cl = 0, store_late = 0, late_after_linger = 0; + + lock_get(pull_lock); + sl = pull_slot_get(id); + /* the echoed key must match the slot's, or this is an answer to a + * request that has already been recycled */ + if (!sl || sl->klen != key->len || memcmp(pull_slot_key(sl), key->s, key->len)) { + lock_release(pull_lock); + LM_DBG("late or unmatched pull reply (id %u) from node %d\n", + id, src_node); + return; + } + + /* record who answered before any dedupe or early return, so the peer + * view reflects what actually arrived on the wire */ + peer_note_reply(src_node, found == PCACHE_FOUND_YES); + + /* Count each node once, whatever the transport does. An id outside + * the range the bitmap covers cannot be tracked, and counting it + * undeduped is exactly the defect the bitmap exists to prevent - two + * answers from one node reaching @expect and manufacturing an absence + * nobody stated. The controller assigns 1..CL_MAX_NODE_ID, but a + * stock clusterer takes whatever the database says, so this is + * reachable without the controller. Drop such a reply rather than + * let it vote. */ + if (src_node <= 0 || src_node > CL_MAX_NODE_ID) { + lock_release(pull_lock); + LM_ERR("pull reply from node id %d, outside 1..%d - cannot be " + "tracked, ignored\n", src_node, CL_MAX_NODE_ID); + return; + } + { + int byte = (src_node - 1) / 8, bit = 1 << ((src_node - 1) % 8); + + if (sl->answered[byte] & bit) { + lock_release(pull_lock); + LM_DBG("duplicate pull reply from node %d - ignored\n", src_node); + return; + } + sl->answered[byte] |= bit; + } + + if (found == PCACHE_FOUND_NO) { + sl->negative++; + __sync_fetch_and_add(&pull_stats[PULL_ST_NEGATIVE], 1); + } else if (found == PCACHE_FOUND_OVERSIZE) { + __sync_fetch_and_add(&pull_stats[PULL_ST_OVERSIZE], 1); + /* someone HAS it - so the key is not absent, whatever the rest of + * the cluster says. Not a negative, and not a value either. */ + sl->oversize = 1; + } else if (!sl->done && val->len <= pull_max_value) { + memcpy(pull_slot_val(sl), val->s, val->len); + sl->vlen = val->len; + /* back to an absolute deadline on our own clock */ + sl->expires = ttl_left ? get_ticks() + (unsigned int)ttl_left : 0; + sl->done = 1; + __sync_fetch_and_add(&pull_stats[PULL_ST_RECEIVED], 1); + } + /* Nobody is waiting on an orphan - its caller timed out and left. We + * are the last chance this value has to reach the cache, so copy what + * we need out of the slot, hand the slot back, and store after the + * lock is dropped. Storing here under pull_lock would nest it outside + * the bucket locks; see the note in pcache_pull_finish(). */ + if (sl->orphan && sl->done) { + late_len = sl->vlen; + late_exp = sl->expires; + late_kl = sl->klen; + late_cl = sl->collen; + memcpy(late_key, pull_slot_key(sl), late_kl); + memcpy(late_col, sl->col, late_cl); + memcpy(late_val, pull_slot_val(sl), late_len); + /* No in-flight TTL correction, deliberately. The peer computes + * ttl_left immediately before sending (see the serve path: it reads + * get_ticks() and hands the value straight to send_rpl), so a peer + * that was busy for seconds still reports a CURRENT remaining TTL. + * The only unaccounted time is the transit back to us. Charging the + * requester's elapsed time here would subtract the peer's own delay + * from a figure that never included it - expiring late answers early + * for precisely the reason they were late. */ + late_after_linger = pull_linger_ms > 0 && + get_uticks() > sl->deadline + (utime_t)pull_linger_ms * 1000; + sl->id = 0; /* back to the pool, job finished */ + sl->orphan = 0; + store_late = 1; + } else if (sl->efd >= 0 && + (sl->done || sl->oversize || sl->negative >= sl->expect)) { + /* Wake whoever is waiting on this slot. The reply almost never + * lands in the process that asked, so this is the only way back + * to it: the fd was created before the fork, which is what lets + * a sibling write to it at all. */ + uint64_t one = 1; + + if (write(sl->efd, &one, sizeof one) != sizeof one) + LM_DBG("could not signal the pull waiter\n"); + } + lock_release(pull_lock); + + if (store_late) { + pcache_col_t *lcol; + str lk, lv, cn; + + lk.s = late_key; lk.len = late_kl; + lv.s = late_val; lv.len = late_len; + cn.s = late_col; cn.len = late_cl; + lcol = col_by_name(&cn); + if (!lcol || !lcol->htable) { + LM_DBG("late pull answer for a collection that went away\n"); + } else if (late_after_linger) { + __sync_fetch_and_add(&pull_stats[PULL_ST_LATE_EXPIRED], 1); + LM_DBG("pull answer for <%.*s> arrived %d ms past the linger " + "window - not stored\n", lk.len, lk.s, pull_linger_ms); + } else if (late_exp && late_exp <= get_ticks()) { + LM_DBG("late pull answer for <%.*s> had already expired\n", + lk.len, lk.s); + } else if (pcache_ht_probe(lcol->htable, &lk, NULL, NULL, NULL) == 0) { + /* Something live is already here. This is read REPAIR: fill what + * is missing, never overwrite what is present. Seconds may have + * passed since the request went out, and a local write in that + * window is by definition fresher than a peer's copy of what we + * asked for. probe() is allocation-free and returns exactly 0 + * for a present, live key - NOT `!= -2`, which would read a pkg + * failure (-1) as "present" and silently stop repairing under the + * very memory pressure that matters, while miscounting it here. */ + __sync_fetch_and_add(&pull_stats[PULL_ST_LATE_SUPERSEDED], 1); + LM_DBG("late pull answer for <%.*s> superseded by a local " + "write - not stored\n", lk.len, lk.s); + } else if (pcache_ht_store(lcol->htable, &lk, &lv, late_exp) < 0) { + LM_ERR("could not store the late pulled value for <%.*s>\n", + lk.len, lk.s); + } else { + __sync_fetch_and_add(&pull_stats[PULL_ST_STORED], 1); + __sync_fetch_and_add(&pull_stats[PULL_ST_LATE_STORED], 1); + __sync_fetch_and_add(&lcol->pulled_in, 1); + pcache_neg_clear(lcol, &lk); + LM_DBG("stored a pull answer that arrived after its caller " + "gave up: <%.*s>\n", lk.len, lk.s); + } + } +} + +/* Reclaim pulls that never got a conclusive answer. + * + * pcache_pull_do_reply() only arms the eventfd once the outcome is settled + * - a value, an oversize holder, or every asked peer having said no. When + * fewer than @expect peers answer (a reply is lost, a peer dies mid-flight, + * a node is asked that never responds) that never becomes true, so on the + * ASYNCHRONOUS path nothing wakes the caller: its resume never runs, + * pcache_pull_finish() is never reached, and since that is the only place a + * slot is released the slot is held for ever. Enough of those and every + * slot is busy and the node stops pulling entirely. + * + * The blocking entry point never had this problem - it polls for at most + * pull_timeout_ms and then calls finish() regardless - which is exactly why + * the concurrent soak, which drives that path, reported no leak. + * + * Two stages, deliberately: + * 1. past its deadline, arm the eventfd once. The caller then resumes + * normally and finish() draws the ordinary "no answer" conclusion and + * counts the timeout, so nothing about the outcome is special-cased + * here. + * 2. still busy well past that, give up on the caller ever coming back + * (its transaction may already be gone) and release the slot. Safe + * because releasing means clearing @id: a late finish() then simply + * fails to find the slot and reports "already reaped", and ids are + * monotonic so it cannot match a slot that has since been reused. + */ +static void pcache_pull_reap(utime_t ticks, void *param) +{ + utime_t now = get_uticks(); + uint64_t one = 1; + int i, woke = 0, dropped = 0, expired = 0; + + if (!pull_slots || !pull_lock) + return; + + lock_get(pull_lock); + for (i = 0; i < PCACHE_PULL_SLOTS; i++) { + struct pcache_pull_slot *sl = pull_slot_at(i); + + if (!sl->id || now <= sl->deadline) + continue; + + if (!sl->reaped) { + sl->reaped = 1; + /* An orphan's caller already finished and left - there is + * nobody on the eventfd, and arming it would leave a count + * for whoever inherits this slot to drain. */ + if (sl->orphan) + continue; + if (sl->efd >= 0 && + write(sl->efd, &one, sizeof one) != sizeof one) + LM_DBG("could not wake the pull waiter on reap\n"); + woke++; + } else if (now > sl->deadline + PCACHE_PULL_ABANDON_US) { + /* An orphan reaching here is the ordinary end of a timeout + * whose answer never came. Only a slot whose caller never + * came back is genuinely abandoned - that distinction is the + * whole point of PULL_ST_ABANDONED and its warning. */ + if (sl->orphan) + expired++; + else + dropped++; + sl->id = 0; + sl->orphan = 0; + } + } + lock_release(pull_lock); + + if (woke) + LM_DBG("reaped %d pull(s) past their deadline\n", woke); + if (expired) + __sync_fetch_and_add(&pull_stats[PULL_ST_ORPHAN_EXPIRED], expired); + if (dropped) { + __sync_fetch_and_add(&pull_stats[PULL_ST_ABANDONED], dropped); + LM_WARN("released %d pull slot(s) whose caller never collected " + "them - a suspended lookup was torn down before it could " + "resume\n", dropped); + } +} + +/* BIN framing -> the shared reply path */ +static void pcache_pull_reply(bin_packet_t *in) +{ + str key, val; + unsigned int id; + int found = 0, ttl_left = 0; + + if (bin_pop_int(in, (int *)&id) < 0 || bin_pop_str(in, &key) < 0 || + bin_pop_int(in, &found) < 0 || bin_pop_int(in, &ttl_left) < 0 || + bin_pop_str(in, &val) < 0) { + LM_ERR("malformed pull reply from node %d\n", in->src_id); + return; + } + pcache_pull_do_reply(in->src_id, id, &key, found, ttl_left, &val); +} + +#ifdef CLUSTERER_CTRL_SUPPORT +/* Controller-plane framing -> the shared paths. Runs in the controller's + * receiving process; the cache is in shm, so serving from here is fine. */ +/* A pull message that arrived on a cluster this module does not sync on. + * + * Rate-limited on the same reasoning as pull_send_failed(): the condition is + * either permanent (a misconfigured sync_cluster_id, in which case EVERY + * message trips it) or routine (a multi-cluster node seeing its other + * clusters' traffic), and an unbounded warn on either is its own incident. + * First occurrence warns, then at most once per interval carrying the count + * it stands for. + * + * It is a WARN and not a DBG because the two causes are indistinguishable + * from the outside and one of them is a silent, total failure of cross-node + * pull: if sync_cluster_id names a cluster the controller does not manage, + * this filter discards everything and the only symptom is that pull never + * converges. There is no way to detect that at startup - the controller's + * get_my_node_id() returns 0 both for "unknown cluster" and for "not joined + * yet" (clusterer_controller.c:7693-7699), and mod_init runs pre-fork, long + * before any cluster is joined. So the check has to live here, and it has to + * say enough to tell the two apart. */ +#define PCACHE_PULL_XCLUSTER_WARN_IVL 60 + +static void pull_foreign_cluster(int cluster_id) +{ + unsigned int now = get_ticks(), held; + + if (pull_stats) + __sync_fetch_and_add(&pull_stats[PULL_ST_FOREIGN_CLUSTER], 1); + if (!pull_xcluster_warn) + return; + + if (pull_xcluster_warn->last != 0 && + now - pull_xcluster_warn->last < PCACHE_PULL_XCLUSTER_WARN_IVL) { + __sync_fetch_and_add(&pull_xcluster_warn->suppressed, 1); + return; + } + pull_xcluster_warn->last = now; + held = __sync_lock_test_and_set(&pull_xcluster_warn->suppressed, 0); + + if (held) + LM_WARN("cross-node pull: dropped a message that arrived on " + "cluster %d but this cache syncs on cluster %d, and %u more " + "in the last %ds - expected on a node that belongs to " + "several controller clusters; if it is EVERY message then " + "sync_cluster_id names a cluster the controller does not " + "manage and pull will never converge (running total: the " + "pulls_foreign_cluster statistic)\n", + cluster_id, sync_cluster_id, held, + PCACHE_PULL_XCLUSTER_WARN_IVL); + else + LM_WARN("cross-node pull: dropped a message that arrived on " + "cluster %d but this cache syncs on cluster %d - expected " + "on a node that belongs to several controller clusters; if " + "it is EVERY message then sync_cluster_id names a cluster " + "the controller does not manage and pull will never " + "converge (running total: the pulls_foreign_cluster " + "statistic)\n", + cluster_id, sync_cluster_id); +} + +static void pcache_clctr_recv(int cluster_id, int src_node_id, str *channel, + str *payload) +{ + const char *p = payload->s; + int left = payload->len; + uint32_t id_be, ttl_be; + uint16_t l16; + str coll, key, val; + unsigned char type; + int found; + + /* Honour the cluster the message arrived on. This is the ONLY place the + * controller plane can be scoped: register_channel() takes a name and a + * callback and nothing else (clusterer_controller/api.h), cl_ctr_channels[] + * is one global table, and cl_ctr_consumer_dispatch() matches on channel + * NAME alone before passing cl->cluster_id here + * (clusterer_controller.c:7388-7402). The channel name is the fixed + * literal "cdbperf-pull", so every cachedb_perf in every cluster shares + * it. + * + * Serving a foreign cluster's request is wrong twice over: the value + * comes out of THIS cache, which is not the one that cluster is + * converging, and the reply is unicast with send_ucast(sync_cluster_id, + * src_node_id) - a node id from the sender's cluster used to address + * sync_cluster_id's members, where the same number is a different + * machine. Symmetrically an inbound reply could satisfy a pending local + * pull with another cluster's value and credit another cluster's node in + * peer_stats. + * + * This is configuration, not misuse: CL_CTR_MAX_CLUSTERS is 16, and the + * documented hybrid topology has native and controller-managed clusters + * side by side on one node with distinct ids. The BIN transport never had + * the hole - clusterer_api.register_capability() binds delivery to + * sync_cluster_id, so scoping happens before the callback. */ + if (cluster_id != sync_cluster_id) { + pull_foreign_cluster(cluster_id); + return; + } + + if (left < 6) + goto bad; + type = (unsigned char)*p++; left--; + memcpy(&id_be, p, 4); p += 4; left -= 4; + + if (type == PCACHE_CLCTR_REQ) { + if (left < 1) + goto bad; + coll.len = (unsigned char)*p++; left--; + if (left < coll.len + 2) + goto bad; + coll.s = (char *)p; p += coll.len; left -= coll.len; + memcpy(&l16, p, 2); p += 2; left -= 2; + key.len = ntohs(l16); + if (left < key.len) + goto bad; + key.s = (char *)p; + pcache_pull_do_serve(src_node_id, ntohl(id_be), &coll, &key, 1); + return; + } + if (type == PCACHE_CLCTR_RPL) { + if (left < 7) + goto bad; + found = (unsigned char)*p++; left--; + memcpy(&ttl_be, p, 4); p += 4; left -= 4; + memcpy(&l16, p, 2); p += 2; left -= 2; + key.len = ntohs(l16); + if (left < key.len + 2) + goto bad; + key.s = (char *)p; p += key.len; left -= key.len; + memcpy(&l16, p, 2); p += 2; left -= 2; + val.len = ntohs(l16); + if (left < val.len) + goto bad; + val.s = (char *)p; + pcache_pull_do_reply(src_node_id, ntohl(id_be), &key, found, + (int)ntohl(ttl_be), &val); + return; + } +bad: + LM_ERR("malformed pull message from node %d on <%.*s>\n", src_node_id, + channel->len, channel->s); +} +#endif + +/* ---- asynchronous face ------------------------------------------------- + * + * Same protocol, without owning a process while the cluster thinks. The + * caller starts a pull, gets back a file descriptor, hands it to whatever + * reactor it lives under, and collects the answer when that fd fires. + * + * The fd is the slot's, created before the fork; the reply handler writes + * to it from whichever process received the answer. Nothing else about + * the protocol changes - the blocking entry point below is this same + * machinery with a poll loop where the reactor would be. + * ---------------------------------------------------------------------- */ + +/* Begin a pull. @fd receives the descriptor to wait on, @id the handle to + * finish with. + * @return 1 = started, wait on @fd, + * 0 = answered without asking anyone (a cached negative), + * -1 = cannot pull (not enabled, no peers, no free slot). */ +/* @hint_node: ask this one node instead of everybody, when membership + * confirms it exists and is not us. A hint is never authoritative - the + * node may have restarted, expired the entry, or had its id reissued to + * somebody else - so an unhelpful answer must leave the caller able to + * ask the rest, which is why a hinted request that comes back empty is + * reported as "no answer" rather than as absence. */ +static int pcache_pull_start(pcache_col_t *col, const str *key, int hint_node, + int *fd, unsigned int *id_out) +{ + struct pcache_pull_slot *sl = NULL; + bin_packet_t packet; + int ids[CL_MAX_NODE_ID], nmembers, i, truncated = 0; + unsigned int gen = 0, id; + + if (!pcache_pull_enabled(col)) { + __sync_fetch_and_add(&pull_stats[PULL_ST_SKIP_NOTREPLICATED], 1); + return -1; + } + if (key->len > pull_max_key || col->col_name.len > 63) { + __sync_fetch_and_add(&pull_stats[PULL_ST_SKIP_TOOLONG], 1); + return -1; + } + if (pcache_neg_check(col, key)) { + __sync_fetch_and_add(&pull_stats[PULL_ST_SUPPRESSED], 1); + return 0; + } + nmembers = pcache_cluster_members(ids, CL_MAX_NODE_ID, &gen, &truncated); + if (nmembers <= 0) { + __sync_fetch_and_add(&pull_stats[PULL_ST_SKIP_NOPEERS], 1); + return -1; + } + + /* Validate the hint before trusting it: a node id that is not a + * current peer is stale, reissued, or simply wrong, and asking it + * would waste the request. */ + if (hint_node > 0) { + int k, live = 0; + + for (k = 0; k < nmembers; k++) + if (ids[k] == hint_node) { + live = 1; + break; + } + if (!live) { + LM_DBG("hint points at node %d, which is not a current peer - " + "asking everybody instead\n", hint_node); + hint_node = 0; + } + } + + lock_get(pull_lock); + for (i = 0; i < PCACHE_PULL_SLOTS; i++) + if (!pull_slot_at(i)->id) { + sl = pull_slot_at(i); + break; + } + if (!sl) { + /* Nothing free. An orphan is only holding its slot on the chance + * that a late answer still arrives, which is worth strictly less + * than the request in front of us - take the one whose deadline + * passed longest ago. A live pull is never stolen. */ + struct pcache_pull_slot *victim = NULL; + int v; + + for (v = 0; v < PCACHE_PULL_SLOTS; v++) { + struct pcache_pull_slot *c = pull_slot_at(v); + + if (c->id && c->orphan && + (!victim || c->deadline < victim->deadline)) + victim = c; + } + if (victim) { + sl = victim; + __sync_fetch_and_add(&pull_stats[PULL_ST_ORPHAN_EVICTED], 1); + } + } + if (!sl) { + lock_release(pull_lock); + __sync_fetch_and_add(&pull_stats[PULL_ST_SKIP_NOSLOT], 1); + LM_WARN("all %d pull slots busy - dropping the request\n", + PCACHE_PULL_SLOTS); + return -1; + } + id = ++(*pull_next_id); + if (!id) + id = ++(*pull_next_id); + { + int efd = sl->efd; /* survives the memset below */ + uint64_t drain; + + memset(sl, 0, sizeof *sl); + sl->efd = efd; + /* a previous user may have left the counter armed if it timed + * out just as an answer arrived - start from a known state */ + while (read(efd, &drain, sizeof drain) == (ssize_t)sizeof drain) + ; + } + sl->id = id; + sl->gen = gen; + sl->hinted = hint_node; + /* A broadcast goes to every peer, but only the ones that fitted the + * snapshot were counted - so on a truncated set the negatives can + * reach @expect while peers nobody tallied still hold the key. */ + sl->partial = hint_node > 0 ? 0 : truncated; + /* one node was asked, so one answer settles it */ + sl->expect = hint_node > 0 ? 1 : nmembers; + sl->deadline = get_uticks() + (utime_t)pull_timeout_ms * 1000; + memcpy(pull_slot_key(sl), key->s, key->len); + sl->klen = key->len; + memcpy(sl->col, col->col_name.s, col->col_name.len); + sl->collen = col->col_name.len; + lock_release(pull_lock); + + __sync_fetch_and_add(&pull_stats[PULL_ST_REQUESTED], 1); +#ifdef CLUSTERER_CTRL_SUPPORT + if (pull_via_clctr) { + char buf[CLCTR_MAX_PAYLOAD]; + str pl; + uint32_t id_be = htonl(id); + uint16_t kl = htons((uint16_t)key->len); + int n = 0; + + /* the entry checks above bound both lengths, so this can only + * fire if those ever change - which is exactly when it should */ + if (PCACHE_CLCTR_REQ_HDR + col->col_name.len + key->len > + (int)sizeof buf) { + LM_ERR("pull request for a %d byte key does not fit %d\n", + key->len, (int)sizeof buf); + goto fail; + } + buf[n++] = PCACHE_CLCTR_REQ; + memcpy(buf + n, &id_be, 4); n += 4; + buf[n++] = (char)col->col_name.len; + memcpy(buf + n, col->col_name.s, col->col_name.len); + n += col->col_name.len; + memcpy(buf + n, &kl, 2); n += 2; + memcpy(buf + n, key->s, key->len); n += key->len; + pl.s = buf; + pl.len = n; + /* one packet, whatever the cluster size - and encrypted, which + * the BIN links are not */ + if (hint_node > 0 + ? clctr_api.send_ucast(sync_cluster_id, hint_node, + &pull_channel, &pl, 0) < 0 + : clctr_api.send_mcast(sync_cluster_id, &pull_channel, + &pl, 0) < 0) + pull_send_failed("a request could not be sent", hint_node); + } else +#endif + { + if (bin_init(&packet, &pcache_sync_cap, PCACHE_PULL_REQ, + PCACHE_SYNC_VERSION, 0) < 0) + goto fail; + if (bin_push_int(&packet, (int)id) < 0 || + bin_push_str(&packet, &col->col_name) < 0 || + bin_push_str(&packet, (str *)key) < 0) { + bin_free_packet(&packet); + goto fail; + } + if ((hint_node > 0 + ? clusterer_api.send_to(&packet, sync_cluster_id, hint_node) + : clusterer_api.send_all(&packet, sync_cluster_id)) != + CLUSTERER_SEND_SUCCESS) + pull_send_failed("a request reached no or only some nodes", + hint_node); + bin_free_packet(&packet); + } + + *fd = sl->efd; + *id_out = id; + return 1; + +fail: + lock_get(pull_lock); + sl->id = 0; + lock_release(pull_lock); + return -1; +} + +/* Collect a started pull. Safe to call on a timeout as well - it releases + * the slot either way, so a caller that gives up leaks nothing. + * @return 1 = value in @out, 0 = definitively absent, -1 = no answer. */ +static int pcache_pull_finish(pcache_col_t *col, const str *key, + unsigned int id, char *out, unsigned int outlen, unsigned int *vlen, + unsigned int *expires) +{ + struct pcache_pull_slot *sl; + unsigned int exp = 0; + uint64_t drain; + int rc = -1; + + lock_get(pull_lock); + sl = pull_slot_get(id); + if (!sl) { + lock_release(pull_lock); + return -1; /* already reaped */ + } + while (read(sl->efd, &drain, sizeof drain) == (ssize_t)sizeof drain) + ; + if (sl->done && sl->vlen <= outlen) { + memcpy(out, pull_slot_val(sl), sl->vlen); + *vlen = sl->vlen; + exp = sl->expires; + if (expires) + *expires = exp; + rc = 1; + } else if (sl->oversize) { + /* a peer holds it but could not send it over this transport. The + * key exists, so this is "no answer", never absence - and nothing + * about it is worth remembering as a negative. */ + rc = -1; + } else if (sl->negative >= sl->expect) { + /* One node was asked and it does not have it. That is not the + * cluster's answer, so it must not become one: report no answer + * and let the caller ask properly. Same for a set we could only + * partly account for - silence from peers we never counted is + * not evidence of absence. */ + rc = (sl->hinted || sl->partial) ? -1 : 0; + } else { + __sync_fetch_and_add(&pull_stats[PULL_ST_TIMEOUT], 1); + } + if (rc == 0 && pc_view && pc_view->generation != sl->gen) { + LM_DBG("membership changed during the pull - not concluding " + "absence\n"); + rc = -1; + } + /* Hand the slot to the protocol rather than the pool when we leave + * empty-handed: the answer may simply be late, and this slot holds the + * only copy of the collection and key it belongs to. rc == 1 is already + * stored below; an oversize holder and a settled absence are final + * answers - none of those wants a late reply. */ + if (rc == 1 || sl->oversize || sl->negative >= sl->expect) { + sl->id = 0; /* the slot is reusable from here */ + } else { + sl->orphan = 1; + __sync_fetch_and_add(&pull_stats[PULL_ST_ORPHANED], 1); + } + lock_release(pull_lock); + + /* Everything below runs OUTSIDE the pull lock, on the copy taken above. + * Storing under it would serialise every node-wide pull behind one + * table write - and worse, it would nest the pull lock outside the + * bucket locks, so any future caller that pulls while holding a bucket + * would deadlock. Nothing here needs the slot. */ + if (rc == 1) { + str v; + + v.s = out; + v.len = *vlen; + if (exp && exp <= get_ticks()) { + LM_DBG("pulled <%.*s> had already expired in flight - not " + "stored\n", key->len, key->s); + } else if (pcache_ht_store(col->htable, key, &v, exp) < 0) { + LM_ERR("could not store the pulled value for <%.*s>\n", + key->len, key->s); + } else { + __sync_fetch_and_add(&pull_stats[PULL_ST_STORED], 1); + /* per-collection twin of PULL_ST_STORED: this is the number + * that actually answers "is this collection converging?" */ + __sync_fetch_and_add(&col->pulled_in, 1); + pcache_neg_clear(col, key); + } + } else if (rc == 0) { + pcache_neg_add(col, key); + } + return rc; +} + +/* Ask the cluster for one key and wait for the answer. + * + * A thin wrapper over the asynchronous pair above, with a poll where a + * reactor would be - so the two paths cannot drift apart, and everything + * that exercises this also exercises the machinery a suspended lookup + * will use. Blocking is why pull_on_miss is off by default. + * + * @return 1 = value found (copied into @out), 0 = definitively absent, + * -1 = no answer in time, or not usable. */ +static int pcache_pull_key(pcache_col_t *col, const str *key, char *out, + unsigned int outlen, unsigned int *vlen, unsigned int *expires) +{ + struct pollfd pfd; + unsigned int id = 0; + int fd = -1, rc, left = pull_timeout_ms; + + rc = pcache_pull_start(col, key, 0, &fd, &id); + if (rc <= 0) + return rc == 0 ? 0 : -1; /* cached negative, or cannot ask */ + + pfd.fd = fd; + pfd.events = POLLIN; + while (left > 0) { + int n = poll(&pfd, 1, left); + + if (n > 0) + break; /* an answer landed */ + if (n < 0 && errno == EINTR) { + left -= 1; /* a signal, not an answer */ + continue; + } + break; /* timeout, or poll failed */ + } + + return pcache_pull_finish(col, key, id, out, outlen, vlen, expires); +} + +/* Ask every peer for a key that cannot exist, purely to see who answers. + * + * The passive per-peer counters cannot separate "this peer ignores us" from + * "we have never had reason to ask it" - both read as zero replies. This + * settles it by generating the traffic itself, over the configured + * transport and through the same serve path a real pull uses, so a peer + * that answers here is genuinely reachable for pulls. + * + * CAVEAT, measured: the request inherits the transport's send semantics. + * Over `bin` that is a TCP write through the clusterer, and a peer that is + * up but not READING (wedged, stopped, swapping) can block it well past + * pull_timeout_ms - the timeout here bounds the wait for an ANSWER, not + * the send. Observed blocking until the peer was resumed. Over `clctr` + * the send is a datagram and cannot block, so this is dependable exactly + * where it is most wanted. Run it on a bin cluster knowing it may stall + * against the kind of peer you are probing for. + * + * @seen must have room for CL_MAX_NODE_ID + 1 flags; on return each live + * peer's slot is 1 if it answered. Returns the number that did, or -1 if + * the pull could not even be started. + */ +static int pcache_cluster_probe(pcache_col_t *col, unsigned char *seen, + int *asked) +{ + struct pollfd pfd; + struct pcache_pull_slot *sl; + unsigned int id = 0; + int fd = -1, rc, left = pull_timeout_ms, i, answered = 0; + /* no caller can store this: perf_set rejects an empty key, and the + * marker byte cannot appear in a th key or any script key */ + static str probe_key = str_init("\x01""cachedb-perf-probe"); + + if (asked) + *asked = 0; + /* a cached negative for the probe key would answer without asking + * anyone, which is the one thing this must not do */ + pcache_neg_clear(col, &probe_key); + + rc = pcache_pull_start(col, &probe_key, 0, &fd, &id); + if (rc <= 0) + return -1; + + pfd.fd = fd; + pfd.events = POLLIN; + while (left > 0) { + int n = poll(&pfd, 1, left); + + if (n > 0) + break; + if (n < 0 && errno == EINTR) { + left -= 1; + continue; + } + break; + } + + /* read the bitmap the reply handler filled in, then release the slot + * exactly as finish() would - the answers are the result here, so the + * value path is not used at all */ + lock_get(pull_lock); + sl = pull_slot_get(id); + if (sl) { + if (asked) + *asked = sl->expect; + for (i = 1; i <= CL_MAX_NODE_ID; i++) { + int byte = (i - 1) / 8, bit = 1 << ((i - 1) % 8); + + if (sl->answered[byte] & bit) { + seen[i] = 1; + answered++; + } + } + sl->id = 0; + } + lock_release(pull_lock); + + /* the probe key is absent everywhere by construction; do not let that + * conclusion linger and suppress the next probe */ + pcache_neg_clear(col, &probe_key); + return answered; +} + +/* perf_cluster_probe [collection] - who is actually reachable for a pull */ +static mi_response_t *do_perf_cluster_probe(str *col_s) +{ + mi_response_t *resp; + mi_item_t *obj, *arr; + pcache_col_t *col; + clusterer_node_t *list, *n; + unsigned char seen[CL_MAX_NODE_ID + 1]; + int answered, asked = 0; + /* MI_SSTR expands to two arguments, so it cannot go in a ternary */ + const char *tname = pull_via_clctr ? "clctr" : "bin"; + + col = col_s ? col_by_name(col_s) : pcache_default_col; + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + if (!pcache_pull_enabled(col)) + return init_mi_error(400, MI_SSTR("cross-node pull is not active " + "for this collection (replicate_collections)")); + + memset(seen, 0, sizeof seen); + answered = pcache_cluster_probe(col, seen, &asked); + if (answered < 0) + return init_mi_error(500, MI_SSTR("could not start the probe - no " + "peers, or no free pull slot")); + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_number(obj, MI_SSTR("asked"), asked) < 0 || + add_mi_number(obj, MI_SSTR("answered"), answered) < 0 || + add_mi_number(obj, MI_SSTR("timeout_ms"), pull_timeout_ms) < 0 || + add_mi_string(obj, MI_SSTR("transport"), tname, strlen(tname)) < 0) + goto err; + + arr = add_mi_array(obj, MI_SSTR("peers")); + if (!arr) + goto err; + list = clusterer_api.get_nodes(sync_cluster_id); + for (n = list; n; n = n->next) { + mi_item_t *p = add_mi_object(arr, NULL, 0); + int up = n->node_id > 0 && n->node_id <= CL_MAX_NODE_ID && + seen[n->node_id]; + + if (!p) { + clusterer_api.free_nodes(list); + goto err; + } + if (add_mi_number(p, MI_SSTR("node_id"), n->node_id) < 0 || + add_mi_string(p, MI_SSTR("answered_probe"), + up ? "yes" : "no", up ? 3 : 2) < 0) { + clusterer_api.free_nodes(list); + goto err; + } + } + if (list) + clusterer_api.free_nodes(list); + return resp; +err: + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("Internal error")); +} + +static mi_response_t *mi_perf_cluster_probe_0(const mi_params_t *params, + struct mi_handler *async) +{ + return do_perf_cluster_probe(NULL); +} + +static mi_response_t *mi_perf_cluster_probe_1(const mi_params_t *params, + struct mi_handler *async) +{ + str col; + + if (get_mi_string_param(params, "collection", &col.s, &col.len) < 0) + return init_mi_param_error(); + return do_perf_cluster_probe(&col); +} + +static void pcache_sync_recv(bin_packet_t *packet) +{ + pcache_col_t *col; + str coll; + + if (packet->type == PCACHE_PULL_REQ) { + pcache_pull_serve(packet); + return; + } + if (packet->type == PCACHE_PULL_RPL) { + pcache_pull_reply(packet); + return; + } + if (packet->type != PCACHE_SYNC_RELOAD) { + LM_WARN("unknown sync packet type %d from node %d\n", + packet->type, packet->src_id); + return; + } + if (bin_pop_str(packet, &coll) < 0) { + LM_ERR("malformed sync packet from node %d\n", packet->src_id); + return; + } + col = col_by_name(&coll); + if (!col || !col->htable) { + LM_WARN("sync for unknown collection <%.*s> from node %d\n", + coll.len, coll.s, packet->src_id); + return; + } + LM_INFO("cluster sync: reloading <%.*s> from DB (issued by node %d)\n", + coll.len, coll.s, packet->src_id); + if (pcache_db_load(col) >= 0) { + col->last_sync_in = get_ticks(); + col->last_sync_src = packet->src_id; + pcache_raise_synced(&col->col_name, packet->src_id); + } +} + +/* broadcast "reload collection X" to the cluster (best-effort - the DB + * already holds the truth; a peer that misses it re-syncs later) */ +static void pcache_sync_broadcast(str *coll) +{ + bin_packet_t packet; + + if (!sync_ready) + return; + if (bin_init(&packet, &pcache_sync_cap, PCACHE_SYNC_RELOAD, + PCACHE_SYNC_VERSION, 0) < 0) { + LM_ERR("failed to init the sync packet\n"); + return; + } + if (bin_push_str(&packet, coll) < 0) { + bin_free_packet(&packet); + return; + } + if (clusterer_api.send_all(&packet, sync_cluster_id) != CLUSTERER_SEND_SUCCESS) + LM_DBG("sync broadcast for <%.*s> reached no/partial nodes\n", + coll->len, coll->s); + bin_free_packet(&packet); +} + +/* save a collection to the DB then signal peers to reload it */ +static int perf_sync_one(pcache_col_t *col, int *bcast) +{ + int rc = pcache_db_save(col); + + if (rc < 0) + return -1; + col->last_sync_out = get_ticks(); + pcache_sync_broadcast(&col->col_name); + if (sync_ready) + (*bcast)++; + return rc; +} + +/* Sharing-tag failover hook (CP-15.12). A BACKUP->ACTIVE flip hands this + * node traffic for state its cache never saw - a mass-miss event. The two + * directions of the hook keep that a snapshot-sized problem: + * ACTIVE - warm the persist collections from the DB snapshot BEFORE the + * storm. On a crash failover the snapshot is the only source + * there is; wall-clock TTLs skip whatever already expired. + * BACKUP - graceful demotion: save our (freshest) state and broadcast, + * so the new active reloads it via the normal sync path. This + * also repairs the flip-ordering race: the new active's warm + * load may run before our save lands, but the broadcast makes + * it reload again afterwards. + * The tag schedules bulk syncs and NOTHING ELSE - lookups are never gated + * on shtag state (a backup node can still legitimately receive traffic). + * Runs in whichever process the clusterer delivers the state change to; + * the DB ops use their own short-lived, fork-safe connections. */ +static void pcache_shtag_cb(str *tag_name, int state, int c_id, void *param) +{ + pcache_col_t *col; + int n = 0, entries = 0, bcast = 0, rc; + + if (state == SHTAG_STATE_ACTIVE) { + /* same scope as a no-argument perf_load/perf_sync: every declared + * collection - the persist flag only governs startup/shutdown */ + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + rc = pcache_db_load(col); + if (rc >= 0) { + n++; + entries += rc; + } + } + LM_INFO("sharing tag <%.*s/%d> ACTIVE: warmed %d collection(s), " + "%d entries, from the DB snapshot\n", + tag_name->len, tag_name->s, c_id, n, entries); + } else if (state == SHTAG_STATE_BACKUP) { + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + if (perf_sync_one(col, &bcast) >= 0) + n++; + } + LM_INFO("sharing tag <%.*s/%d> BACKUP: saved %d collection(s)%s\n", + tag_name->len, tag_name->s, c_id, n, + bcast ? ", peers signalled to reload" : ""); + } +} + +/* perf_sync [collection] - save-then-broadcast; all declared if none named */ +static mi_response_t *do_perf_sync(str *col_s) +{ + pcache_col_t *col; + mi_response_t *resp; + mi_item_t *obj; + int saved = 0, ncol = 0, bcast = 0, rc; + + if (!pcache_db_enabled()) + return init_mi_error(500, + MI_SSTR("no DB backend configured (set db_url)")); + + if (col_s) { + col = col_by_name(col_s); + if (!col) + return init_mi_error(404, MI_SSTR("no such collection")); + rc = perf_sync_one(col, &bcast); + if (rc < 0) + return init_mi_error(500, MI_SSTR("save failed")); + saved = rc; + ncol = 1; + } else { + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + rc = perf_sync_one(col, &bcast); + if (rc < 0) + return init_mi_error(500, MI_SSTR("save failed")); + saved += rc; + ncol++; + } + } + + resp = init_mi_result_object(&obj); + if (!resp) + return NULL; + if (add_mi_number(obj, MI_SSTR("collections"), ncol) < 0 || + add_mi_number(obj, MI_SSTR("saved"), saved) < 0 || + add_mi_number(obj, MI_SSTR("broadcast"), bcast) < 0) + goto err; + if (!sync_ready && add_mi_string(obj, MI_SSTR("note"), + MI_SSTR("cluster sync inactive (no clusterer / cluster_id 0) - " + "saved to the DB only")) < 0) + goto err; + return resp; +err: + free_mi_response(resp); + return init_mi_error(500, MI_SSTR("internal error")); +} + +static int w_perf_sync(struct sip_msg *msg, str *col_s) +{ + pcache_col_t *col; + int bcast = 0; + + if (!pcache_db_enabled()) { + LM_ERR("perf_sync needs a DB backend (db_url)\n"); + return -1; + } + if (col_s) { + col = col_by_name(col_s); + if (!col || perf_sync_one(col, &bcast) < 0) + return -1; + } else { + for (col = pcache_collection; col; col = col->next) + if (col->htable && perf_sync_one(col, &bcast) < 0) + return -1; + } + return 1; +} + +/* thin per-arity recipe wrappers: extract params, then defer to the workers */ +#define MI_S(nm, dst) \ + do { if (get_mi_string_param(params, nm, &(dst).s, &(dst).len) < 0) \ + return init_mi_param_error(); } while (0) +#define MI_I(nm, dst) \ + do { if (get_mi_int_param(params, nm, &(dst)) < 0) \ + return init_mi_param_error(); } while (0) + +static mi_response_t *mi_perf_keys_1(const mi_params_t *params, struct mi_handler *a) +{ str g; MI_S("glob", g); return do_perf_keys(&g, NULL, 0, 0); } +static mi_response_t *mi_perf_keys_2(const mi_params_t *params, struct mi_handler *a) +{ str g, c; MI_S("glob", g); MI_S("collection", c); return do_perf_keys(&g, &c, 0, 0); } +static mi_response_t *mi_perf_keys_3(const mi_params_t *params, struct mi_handler *a) +{ str g, c; int l; MI_S("glob", g); MI_S("collection", c); MI_I("limit", l); + return do_perf_keys(&g, &c, l, 0); } +static mi_response_t *mi_perf_keys_gl(const mi_params_t *params, struct mi_handler *a) +{ str g; int l; MI_S("glob", g); MI_I("limit", l); return do_perf_keys(&g, NULL, l, 0); } + +static mi_response_t *mi_perf_dump_1(const mi_params_t *params, struct mi_handler *a) +{ str g; MI_S("glob", g); return do_perf_keys(&g, NULL, 0, 1); } +static mi_response_t *mi_perf_dump_2(const mi_params_t *params, struct mi_handler *a) +{ str g, c; MI_S("glob", g); MI_S("collection", c); return do_perf_keys(&g, &c, 0, 1); } +static mi_response_t *mi_perf_dump_3(const mi_params_t *params, struct mi_handler *a) +{ str g, c; int l; MI_S("glob", g); MI_S("collection", c); MI_I("limit", l); + return do_perf_keys(&g, &c, l, 1); } +static mi_response_t *mi_perf_dump_gl(const mi_params_t *params, struct mi_handler *a) +{ str g; int l; MI_S("glob", g); MI_I("limit", l); return do_perf_keys(&g, NULL, l, 1); } + +static mi_response_t *mi_perf_scan_1(const mi_params_t *params, struct mi_handler *a) +{ int cu; MI_I("cursor", cu); return do_perf_scan(cu, NULL, 0); } +static mi_response_t *mi_perf_scan_2(const mi_params_t *params, struct mi_handler *a) +{ int cu; str g; MI_I("cursor", cu); MI_S("glob", g); return do_perf_scan(cu, &g, 0); } +static mi_response_t *mi_perf_scan_3(const mi_params_t *params, struct mi_handler *a) +{ int cu, co; str g; MI_I("cursor", cu); MI_S("glob", g); MI_I("count", co); + return do_perf_scan(cu, &g, co); } +static mi_response_t *mi_perf_scan_cc(const mi_params_t *params, struct mi_handler *a) +{ int cu, co; MI_I("cursor", cu); MI_I("count", co); return do_perf_scan(cu, NULL, co); } + +static mi_response_t *mi_perf_pull_1(const mi_params_t *params, struct mi_handler *a) +{ str k; MI_S("key", k); return do_perf_pull(&k, NULL); } +static mi_response_t *mi_perf_pull_2(const mi_params_t *params, struct mi_handler *a) +{ str k, c; MI_S("key", k); MI_S("collection", c); return do_perf_pull(&k, &c); } + +static mi_response_t *mi_perf_probe_1(const mi_params_t *params, struct mi_handler *a) +{ str k; MI_S("key", k); return do_perf_probe(&k, NULL); } +static mi_response_t *mi_perf_probe_2(const mi_params_t *params, struct mi_handler *a) +{ str k, c; MI_S("key", k); MI_S("collection", c); return do_perf_probe(&k, &c); } + +static mi_response_t *mi_perf_get_1(const mi_params_t *params, struct mi_handler *a) +{ str k; MI_S("key", k); return do_perf_get(&k, NULL); } +static mi_response_t *mi_perf_get_2(const mi_params_t *params, struct mi_handler *a) +{ str k, c; MI_S("key", k); MI_S("collection", c); return do_perf_get(&k, &c); } + +static mi_response_t *mi_perf_set_2(const mi_params_t *params, struct mi_handler *a) +{ str k, v; MI_S("key", k); MI_S("value", v); return do_perf_set(&k, &v, 0, NULL); } +static mi_response_t *mi_perf_set_3(const mi_params_t *params, struct mi_handler *a) +{ str k, v; int t; MI_S("key", k); MI_S("value", v); MI_I("ttl", t); + return do_perf_set(&k, &v, t, NULL); } +static mi_response_t *mi_perf_set_4(const mi_params_t *params, struct mi_handler *a) +{ str k, v, c; int t; MI_S("key", k); MI_S("value", v); MI_I("ttl", t); + MI_S("collection", c); return do_perf_set(&k, &v, t, &c); } +static mi_response_t *mi_perf_set_kvc(const mi_params_t *params, struct mi_handler *a) +{ str k, v, c; MI_S("key", k); MI_S("value", v); MI_S("collection", c); + return do_perf_set(&k, &v, 0, &c); } + +static mi_response_t *mi_perf_del_1(const mi_params_t *params, struct mi_handler *a) +{ str g; MI_S("glob", g); return do_perf_del_mi(&g, NULL); } +static mi_response_t *mi_perf_del_2(const mi_params_t *params, struct mi_handler *a) +{ str g, c; MI_S("glob", g); MI_S("collection", c); return do_perf_del_mi(&g, &c); } + +static mi_response_t *mi_perf_ttl_2(const mi_params_t *params, struct mi_handler *a) +{ str g; int t; MI_S("glob", g); MI_I("ttl", t); return do_perf_ttl(&g, t, NULL); } +static mi_response_t *mi_perf_ttl_3(const mi_params_t *params, struct mi_handler *a) +{ str g, c; int t; MI_S("glob", g); MI_I("ttl", t); MI_S("collection", c); + return do_perf_ttl(&g, t, &c); } + +static mi_response_t *mi_perf_save_0(const mi_params_t *params, struct mi_handler *a) +{ return do_perf_persist(NULL, 1); } +static mi_response_t *mi_perf_save_1(const mi_params_t *params, struct mi_handler *a) +{ str c; MI_S("collection", c); return do_perf_persist(&c, 1); } +static mi_response_t *mi_perf_load_0(const mi_params_t *params, struct mi_handler *a) +{ return do_perf_persist(NULL, 0); } +static mi_response_t *mi_perf_load_1(const mi_params_t *params, struct mi_handler *a) +{ str c; MI_S("collection", c); return do_perf_persist(&c, 0); } +static mi_response_t *mi_perf_sync_0(const mi_params_t *params, struct mi_handler *a) +{ return do_perf_sync(NULL); } +static mi_response_t *mi_perf_sync_1(const mi_params_t *params, struct mi_handler *a) +{ str c; MI_S("collection", c); return do_perf_sync(&c); } + +#undef MI_S +#undef MI_I + +static const mi_export_t mi_cmds[] = { + { "perf_stats", "per-collection stats (entries, buckets, load factor, " + "overflow, seqlock retries, memory tier)", 0, 0, { + {mi_perf_stats_1, {0}}, + {mi_perf_stats_2, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_stats_reset", "re-baseline the cumulative counters so the rates " + "cover a fresh interval; live gauges are unaffected", 0, 0, { + {mi_perf_stats_reset_1, {0}}, + {mi_perf_stats_reset_2, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_keys", "names of keys matching a glob, bounded (KEYS-like)", 0, 0, { + {mi_perf_keys_1, {"glob", 0}}, + {mi_perf_keys_2, {"glob", "collection", 0}}, + {mi_perf_keys_gl, {"glob", "limit", 0}}, + {mi_perf_keys_3, {"glob", "collection", "limit", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_scan", "cursor-based incremental iteration (Redis SCAN); pass " + "cursor 0 to start, iteration ends when it returns 0", 0, 0, { + {mi_perf_scan_1, {"cursor", 0}}, + {mi_perf_scan_2, {"cursor", "glob", 0}}, + {mi_perf_scan_cc, {"cursor", "count", 0}}, + {mi_perf_scan_3, {"cursor", "glob", "count", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_dump", "keys AND values matching a glob, bounded (opt-in values)", + 0, 0, { + {mi_perf_dump_1, {"glob", 0}}, + {mi_perf_dump_2, {"glob", "collection", 0}}, + {mi_perf_dump_gl, {"glob", "limit", 0}}, + {mi_perf_dump_3, {"glob", "collection", "limit", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_pull", "fetch one key from the cluster on a local miss", 0, 0, { + {mi_perf_pull_1, {"key", 0}}, + {mi_perf_pull_2, {"key", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_cluster_probe", "ask every peer for a key that cannot exist, to " + "see which ones actually answer a pull", 0, 0, { + {mi_perf_cluster_probe_0, {0}}, + {mi_perf_cluster_probe_1, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_probe", "one key: is it here, its TTL and size - no value", 0, 0, { + {mi_perf_probe_1, {"key", 0}}, + {mi_perf_probe_2, {"key", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_get", "one key: value, TTL and size", 0, 0, { + {mi_perf_get_1, {"key", 0}}, + {mi_perf_get_2, {"key", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_set", "write one key (optional ttl seconds, 0 = never)", 0, 0, { + {mi_perf_set_2, {"key", "value", 0}}, + {mi_perf_set_3, {"key", "value", "ttl", 0}}, + {mi_perf_set_kvc, {"key", "value", "collection", 0}}, + {mi_perf_set_4, {"key", "value", "ttl", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_del", "delete keys matching a glob (the perf_del() script fn)", + 0, 0, { + {mi_perf_del_1, {"glob", 0}}, + {mi_perf_del_2, {"glob", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_ttl", "re-arm the TTL of every key matching a glob (ttl seconds, " + "0 = never); returns the count updated", 0, 0, { + {mi_perf_ttl_2, {"glob", "ttl", 0}}, + {mi_perf_ttl_3, {"glob", "ttl", "collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_save", "snapshot a collection to the DB backend (all declared " + "collections if none is named)", 0, 0, { + {mi_perf_save_0, {0}}, + {mi_perf_save_1, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_load", "load a collection from the DB backend (all declared " + "collections if none is named)", 0, 0, { + {mi_perf_load_0, {0}}, + {mi_perf_load_1, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + { "perf_sync", "save a collection to the DB and signal the cluster to " + "reload it (all declared collections if none is named)", 0, 0, { + {mi_perf_sync_0, {0}}, + {mi_perf_sync_1, {"collection", 0}}, + {EMPTY_MI_RECIPE}}, + {0} + }, + {EMPTY_MI_EXPORT} +}; + +/* soft clusterer dependency: when sync_cluster_id is set, have clusterer + * init first (so register_capability runs and "cachedb-perf-sync" shows in + * clusterer_list_caps) - but SILENT, so a missing clusterer does not abort; + * perf_sync then degrades to a DB save (mod_init handles it) */ +static module_dependency_t *get_deps_sync_cluster(const param_export_t *param) +{ + if (*(int *)param->param_pointer <= 0) + return NULL; + return alloc_module_dep(MOD_TYPE_DEFAULT, "clusterer", DEP_SILENT); +} + +static const dep_export_t deps = { + { /* OpenSIPS module dependencies */ + { MOD_TYPE_NULL, NULL, 0 }, + }, + { /* modparam dependencies */ + { "sync_cluster_id", get_deps_sync_cluster }, + { NULL, NULL }, + }, +}; + +/** module exports */ +struct module_exports exports = { + "cachedb_perf", /* module name */ + MOD_TYPE_CACHEDB, /* class of this module */ + MODULE_VERSION, + DEFAULT_DLFLAGS, /* dlopen flags */ + 0, /* load function */ + &deps, /* OpenSIPS module dependencies */ + cmds, /* exported functions */ + 0, /* exported async functions */ + params, /* exported parameters */ + mod_stats, /* exported statistics */ + mi_cmds, /* exported MI functions */ + 0, /* exported pseudo-variables */ + 0, /* exported transformations */ + 0, /* extra processes */ + 0, /* module pre-initialization function */ + mod_init, /* module initialization function */ + (response_function) 0, /* response handling function */ + (destroy_function) mod_destroy, /* destroy function */ + child_init, /* per-child init function */ + 0 /* reload confirm function */ +}; + + +/* + * connection management + * + * The collection is taken from the URL's "database" part (perf:///name) + * or, as a convenience, from the "host" part (perf://name) - a host has + * no meaning for a local cache. No collection in the URL means the + * default one. Matching is exact and an unknown name is a hard error. + */ +/* Connections this module created, so a con arriving through the exported + * pull API can be recognised before anything is read out of it. + * + * The API is bound by name through find_export, which offers no type safety + * whatever: any module that loads cachedb_perf can call these entry points + * and pass a cachedb_con belonging to some other backend. Reading ->data as + * a pcache_con at that point is a type confusion - the field where this + * module keeps its collection pointer is, in a redis or local connection, + * whatever that module put there. So the pointer is checked against this + * list instead, which never dereferences the stranger. + * + * Per-process (pkg), like the connections themselves, and short - one entry + * per URL this process opened. */ +struct pcache_con_reg { + pcache_con *con; + struct pcache_con_reg *next; +}; +static struct pcache_con_reg *pcache_con_reg_head; + +static void pcache_con_register(pcache_con *c) +{ + struct pcache_con_reg *r = pkg_malloc(sizeof(*r)); + + if (!r) { + LM_ERR("out of pkg memory registering a connection\n"); + return; + } + r->con = c; + r->next = pcache_con_reg_head; + pcache_con_reg_head = r; +} + +static void pcache_con_unregister(pcache_con *c) +{ + struct pcache_con_reg **p = &pcache_con_reg_head; + + while (*p) { + if ((*p)->con == c) { + struct pcache_con_reg *dead = *p; + + *p = dead->next; + pkg_free(dead); + return; + } + p = &(*p)->next; + } +} + +/* The collection behind a connection, or NULL when the connection is not + * ours. Every exported entry point goes through this. */ +static pcache_col_t *pcache_col_of(cachedb_con *con) +{ + struct pcache_con_reg *r; + + if (!con || !con->data) + return NULL; + for (r = pcache_con_reg_head; r; r = r->next) + if (r->con == (pcache_con *)con->data) + return r->con->col; + + LM_ERR("a connection that this module did not open was passed to its " + "pull API - refusing it. The caller is holding a handle to a " + "different cachedb backend; only a perf:// connection can be " + "pulled through\n"); + return NULL; +} + +static pcache_con *pcache_new_connection(struct cachedb_id *id) +{ + pcache_con *con; + pcache_col_t *col; + const char *sel = NULL; + int len; + + if (!id) { + LM_ERR("null cachedb_id\n"); + return NULL; + } + + if (id->database && id->database[0]) { + sel = id->database; + if (id->host && id->host[0] && strcmp(id->host, id->database)) + LM_WARN("URL <%s>: ignoring host part <%s>, " + "using collection <%s>\n", + id->initial_url, id->host, sel); + } else if (id->host && id->host[0]) { + sel = id->host; + } + + if (!sel) + sel = PCACHE_DEFAULT_COLLECTION; + + len = strlen(sel); + for (col = pcache_collection; col; col = col->next) + if (col->col_name.len == len && + !memcmp(col->col_name.s, sel, len)) + break; + + if (!col) { + LM_ERR("collection <%s> is not defined in 'cache_collections'\n", + sel); + return NULL; + } + + con = pkg_malloc(sizeof *con); + if (!con) { + LM_ERR("no more pkg memory\n"); + return NULL; + } + memset(con, 0, sizeof *con); + con->id = id; + con->ref = 1; + con->col = col; + + LM_DBG("URL <%s> bound to collection <%.*s>\n", + id->initial_url, col->col_name.len, col->col_name.s); + + pcache_con_register(con); + return con; +} + +static cachedb_con *pcache_init(str *url) +{ + return cachedb_do_init(url, (void *)pcache_new_connection); +} + +static void pcache_free_connection(cachedb_pool_con *con) +{ + pcache_con_unregister((pcache_con *)con); + + pkg_free(con); +} + +static void pcache_destroy(cachedb_con *con) +{ + cachedb_do_close(con, pcache_free_connection); +} + + +/* + * CP-11 event raising. Every raise is gated by evi_probe_event(), so with + * no subscribers the cost is one shared read and nothing else - none of + * these sit on the lock-free get/set hot path (expiry/growth run in the + * maintenance timer, NOMEM only on a dropped write, degraded once at boot). + */ +static void pcache_on_expired(const str *key, void *ctx) +{ + str *coll = ctx; + evi_params_p list; + + /* the timer already probed before opting this sweep into events */ + list = evi_get_params(); + if (!list) + return; + if (evi_param_add_str(list, &evp_collection, coll) || + evi_param_add_str(list, &evp_key, key)) { + evi_free_params(list); + return; + } + if (evi_raise_event(evi_expired_id, list)) + LM_ERR("failed to raise %.*s\n", evi_expired_name.len, + evi_expired_name.s); +} + +static void pcache_raise_nomem(str *coll, str *key, int size) +{ + evi_params_p list; + + if (evi_nomem_id == EVI_ERROR || !evi_probe_event(evi_nomem_id)) + return; + list = evi_get_params(); + if (!list) + return; + if (evi_param_add_str(list, &evp_collection, coll) || + evi_param_add_str(list, &evp_key, key) || + evi_param_add_int(list, &evp_size, &size)) { + evi_free_params(list); + return; + } + if (evi_raise_event(evi_nomem_id, list)) + LM_ERR("failed to raise %.*s\n", evi_nomem_name.len, + evi_nomem_name.s); +} + +static void pcache_raise_grown(str *coll, int prev_b, int new_b, int splits, + int entries) +{ + evi_params_p list; + + if (evi_grown_id == EVI_ERROR || !evi_probe_event(evi_grown_id)) + return; + list = evi_get_params(); + if (!list) + return; + if (evi_param_add_str(list, &evp_collection, coll) || + evi_param_add_int(list, &evp_prev_buckets, &prev_b) || + evi_param_add_int(list, &evp_buckets, &new_b) || + evi_param_add_int(list, &evp_splits, &splits) || + evi_param_add_int(list, &evp_entries, &entries)) { + evi_free_params(list); + return; + } + if (evi_raise_event(evi_grown_id, list)) + LM_ERR("failed to raise %.*s\n", evi_grown_name.len, + evi_grown_name.s); +} + +static void pcache_raise_degraded(void) +{ + evi_params_p list; + str backing; + int req = pcache_arena_hugepage_mb; + int tier = pcache_arena_tier(); + int oc = pcache_mem.huge_overcommit; + + if (evi_degraded_id == EVI_ERROR || !evi_probe_event(evi_degraded_id)) + return; + list = evi_get_params(); + if (!list) + return; + backing.s = (char *)pcache_mem_tier_str(tier); + backing.len = strlen(backing.s); + if (evi_param_add_int(list, &evp_requested_mb, &req) || + evi_param_add_int(list, &evp_tier, &tier) || + evi_param_add_str(list, &evp_backing, &backing) || + evi_param_add_int(list, &evp_overcommit, &oc)) { + evi_free_params(list); + return; + } + if (evi_raise_event(evi_degraded_id, list)) + LM_ERR("failed to raise %.*s\n", evi_degraded_name.len, + evi_degraded_name.s); +} + +static void pcache_raise_synced(str *coll, int src_node) +{ + evi_params_p list; + + if (evi_synced_id == EVI_ERROR || !evi_probe_event(evi_synced_id)) + return; + list = evi_get_params(); + if (!list) + return; + if (evi_param_add_str(list, &evp_collection, coll) || + evi_param_add_int(list, &evp_source_node, &src_node)) { + evi_free_params(list); + return; + } + if (evi_raise_event(evi_synced_id, list)) + LM_ERR("failed to raise %.*s\n", evi_synced_name.len, + evi_synced_name.s); +} + +/* + * the cachedb vtable (CP-04) - thin adapters over the table core. TTL to + * absolute-ticks conversion happens here; internals only see absolutes. + */ +static pcache_htable_t *con_ht(cachedb_con *con) +{ + pcache_con *c = con ? (pcache_con *)con->data : NULL; + + if (!c || !c->col || !c->col->htable) { + LM_ERR("no connection state\n"); + return NULL; + } + return c->col->htable; +} + +static inline unsigned int ttl_to_abs(int expires) +{ + return expires > 0 ? get_ticks() + (unsigned int)expires : 0; +} + +/* Read repair on the normal get path: a miss here asks the cluster, and a + * value that comes back is returned as if it had been local all along - + * so a consumer gets cross-node lookups without knowing they exist. + * + * Off by default, and it must stay that way until the lookup can suspend + * the transaction instead of the worker: this blocks for as long as the + * pull takes, which on a SIP path means a worker not serving anything + * else. A LAN pull is a couple of milliseconds and the negative cache + * absorbs retransmits, but "usually fast" is not the same as "safe under + * load", which is why the startup warning says so out loud. */ +static int pcache_htable_fetch(cachedb_con *con, str *attr, str *val) +{ + pcache_col_t *col = con ? ((pcache_con *)con->data)->col : NULL; + unsigned int vlen = 0; + int rc; + + if (!col || !col->htable) + return -1; + rc = pcache_ht_fetch(col->htable, attr, val); + if (rc != -2) + return rc; + /* A miss from here on. The two conditions below skip the pull without + * ever entering pcache_pull_start(), so they have to be accounted for + * here or the miss disappears - which is precisely how the gap between + * `misses` and `pulls_requested` came to be unexplainable. + * pull_on_miss off is a deployment choice and not worth its own counter + * (nothing is being refused - the feature is simply not in use), but a + * collection that is not replicated IS worth counting: that is the case + * an operator misreads as "pull is enabled, why is nothing pulling". */ + if (!pull_on_miss) + return rc; + if (!pcache_pull_enabled(col)) { + __sync_fetch_and_add(&pull_stats[PULL_ST_SKIP_NOTREPLICATED], 1); + return rc; + } + + /* the pull buffer lives in this branch, not in the frame of every + * local hit: this is the vtable read, and a cross-node miss is the + * rare path */ + { + char buf[PCACHE_PULL_MAX_VAL]; + + if (pcache_pull_key(col, attr, buf, sizeof buf, &vlen, NULL) != 1) + return -2; /* absent, or nobody answered */ + + /* hand back a copy the caller owns, exactly as a local hit would */ + val->s = pkg_malloc(vlen ? vlen : 1); + if (!val->s) { + LM_ERR("no more pkg memory for a %u byte pulled value\n", vlen); + return -1; + } + memcpy(val->s, buf, vlen); + } + val->len = vlen; + return 0; +} + +/* CACHEDB_CAP_GET_BUF: the allocation-free read. Note this deliberately + * does NOT touch pcache_htable_fetch() above - the vtable get() keeps its + * own documented behaviour, byte for byte. */ +static int pcache_htable_fetch_buf(cachedb_con *con, str *attr, char *buf, + unsigned int buflen, unsigned int *vlen, unsigned int *needed) +{ + pcache_htable_t *ht = con_ht(con); + + if (vlen) + *vlen = 0; + if (needed) + *needed = 0; + return ht ? pcache_ht_fetch_buf(ht, attr, buf, buflen, vlen, needed) : -1; +} + +static int pcache_htable_fetch_counter(cachedb_con *con, str *attr, int *val) +{ + pcache_htable_t *ht = con_ht(con); + long long ll; + str v; + int rc; + + if (!ht) + return -1; + rc = pcache_ht_fetch(ht, attr, &v); + if (rc != 0) + return rc; /* -2 absent / -1 error */ + rc = pcache_str2ll(v.s, v.len, &ll); + pkg_free(v.s); + if (rc < 0) { + LM_ERR("value of <%.*s> is not a counter\n", attr->len, attr->s); + return -1; + } + if (val) + *val = (int)ll; + return 0; +} + +static int pcache_htable_insert(cachedb_con *con, str *attr, str *val, + int expires) +{ + pcache_col_t *col = con ? ((pcache_con *)con->data)->col : NULL; + int rc; + + if (!col || !col->htable) + return -1; + rc = pcache_ht_store(col->htable, attr, val, ttl_to_abs(expires)); + if (rc == -2) { /* arena full - write dropped */ + pcache_raise_nomem(&col->col_name, attr, val ? val->len : 0); + return -1; + } + /* the key exists here now, so any conclusion we drew about the + * cluster not having it no longer describes it */ + if (rc >= 0) + pcache_neg_clear(col, attr); + return rc; +} + +static int pcache_htable_remove(cachedb_con *con, str *attr) +{ + pcache_htable_t *ht = con_ht(con); + + if (!ht) + return -1; + return pcache_ht_remove(ht, attr) < 0 ? -1 : 0; +} + +static int pcache_htable_add(cachedb_con *con, str *attr, int val, + int expires, int *new_val) +{ + pcache_htable_t *ht = con_ht(con); + long long nv; + + if (!ht) + return -1; + if (pcache_ht_add(ht, attr, val, ttl_to_abs(expires), &nv) < 0) + return -1; + if (new_val) + *new_val = (int)nv; + return 0; +} + +static int pcache_htable_sub(cachedb_con *con, str *attr, int val, + int expires, int *new_val) +{ + pcache_htable_t *ht = con_ht(con); + long long nv; + + if (!ht) + return -1; + if (pcache_ht_add(ht, attr, -(long long)val, ttl_to_abs(expires), + &nv) < 0) + return -1; + if (new_val) + *new_val = (int)nv; + return 0; +} + +struct iter_ctx { + int (*kv)(const str *key, const str *value); + unsigned int now; +}; + +static int iter_adapt_cb(const str *key, const str *val, unsigned int exp, + void *p) +{ + struct iter_ctx *ic = p; + + if (exp && exp <= ic->now) + return 0; /* expired-as-absent */ + return ic->kv(key, val); +} + +static int pcache_htable_iter_keys(cachedb_con *con, + int (*kv_func)(const str *key, const str *value)) +{ + pcache_con *c = con ? (pcache_con *)con->data : NULL; + struct iter_ctx ic; + + if (!c || !c->col || !c->col->htable) { + LM_ERR("no connection state\n"); + return -1; + } + ic.kv = kv_func; + ic.now = get_ticks(); + return pcache_ht_iter(c->col->htable, iter_adapt_cb, &ic); +} + + +/* + * glob operations (CP-07): perf_del / perf_mget / perf_mget_json, all on + * the pcache_ht_iter() walker. Redis SCAN-class guarantee: entries + * mutating concurrently may be seen once, twice or not at all. + */ + +static pcache_col_t *col_by_name(const str *name) +{ + pcache_col_t *col; + str def = str_init(PCACHE_DEFAULT_COLLECTION); + + if (!name || !name->s || !name->len) { + /* no collection argument = wherever cache_store("perf", ...) + * goes, i.e. the default connection's collection */ + if (pcache_default_col) + return pcache_default_col; + name = &def; + } + for (col = pcache_collection; col; col = col->next) + if (col->col_name.len == name->len && + !memcmp(col->col_name.s, name->s, name->len)) + return col; + LM_ERR("collection <%.*s> is not defined\n", name->len, name->s); + return NULL; +} + +static int fixup_check_wvar(void **param) +{ + if (((pv_spec_t *)*param)->setf == NULL) { + LM_ERR("output parameter must be a writable variable\n"); + return -1; + } + return 0; +} + +static char *glob_dup(const str *glob) +{ + char *pat = pkg_malloc(glob->len + 1); + + if (!pat) { + LM_ERR("no more pkg memory\n"); + return NULL; + } + memcpy(pat, glob->s, glob->len); + pat[glob->len] = 0; + return pat; +} + +struct del_ctx { + const char *pat; + str *keys; + unsigned int n, cap; + int oom; +}; + +static int del_collect_cb(const str *key, const str *val, unsigned int exp, + void *p) +{ + struct del_ctx *dc = p; + str *grown; + + if (fnmatch(dc->pat, key->s, 0)) + return 0; + if (dc->n == dc->cap) { + dc->cap = dc->cap ? 2 * dc->cap : 64; + grown = pkg_realloc(dc->keys, dc->cap * sizeof *dc->keys); + if (!grown) { + dc->oom = 1; + return -1; + } + dc->keys = grown; + } + if (pkg_str_dup(&dc->keys[dc->n], key) < 0) { + dc->oom = 1; + return -1; + } + dc->n++; + return 0; +} + +/* glob-delete core, shared by the script perf_del() and the MI perf_del: + * collect matches lock-free, then remove one by one - a glob delete is not + * an atomic snapshot (and cannot usefully be). Returns the number removed + * (>= 0), or -1 on OOM (the removal is then partial). */ +static int perf_del_run(pcache_col_t *col, str *glob) +{ + struct del_ctx dc; + char *pat; + unsigned int i, removed = 0; + + pat = glob_dup(glob); + if (!pat) + return -1; + + memset(&dc, 0, sizeof dc); + dc.pat = pat; + pcache_ht_iter(col->htable, del_collect_cb, &dc); + + for (i = 0; i < dc.n; i++) { + if (pcache_ht_remove(col->htable, &dc.keys[i]) == 1) + removed++; + pkg_free(dc.keys[i].s); + } + if (dc.keys) + pkg_free(dc.keys); + + LM_DBG("glob <%s>: removed %u of %u matches\n", pat, removed, dc.n); + pkg_free(pat); + if (dc.oom) { + LM_ERR("out of pkg memory mid-walk - removal is partial\n"); + return -1; + } + return (int)removed; +} + +static int w_perf_del(struct sip_msg *msg, str *glob, str *col_s) +{ + pcache_col_t *col = col_by_name(col_s); + int removed; + + if (!col) + return -1; + removed = perf_del_run(col, glob); + return removed > 0 ? removed : -1; /* 0 matches / OOM -> script-false */ +} + +/* growing pkg buffer for the JSON form */ +struct jbuf { + char *s; + unsigned int len, cap; +}; + +static int jb_put(struct jbuf *jb, const char *p, unsigned int n) +{ + char *grown; + + while (jb->len + n > jb->cap) { + jb->cap = jb->cap ? 2 * jb->cap : 4096; + grown = pkg_realloc(jb->s, jb->cap); + if (!grown) + return -1; + jb->s = grown; + } + memcpy(jb->s + jb->len, p, n); + jb->len += n; + return 0; +} + +/* length-based JSON string emission: escapes quote, backslash and + * control bytes (values may be binary - embedded NULs survive); bytes + * >= 0x80 pass through, so strict-JSON consumers need UTF-8 values */ +static int jb_put_jstr(struct jbuf *jb, const str *s) +{ + static const char hexd[] = "0123456789abcdef"; + char esc[6] = "\\u00"; + unsigned int i, from = 0; + unsigned char c; + int r = jb_put(jb, "\"", 1); + + for (i = 0; i < (unsigned int)s->len && r == 0; i++) { + c = (unsigned char)s->s[i]; + if (c != '"' && c != '\\' && c >= 0x20) + continue; + r = jb_put(jb, s->s + from, i - from); + if (r == 0) { + if (c == '"') + r = jb_put(jb, "\\\"", 2); + else if (c == '\\') + r = jb_put(jb, "\\\\", 2); + else { + esc[4] = hexd[c >> 4]; + esc[5] = hexd[c & 0xF]; + r = jb_put(jb, esc, 6); + } + } + from = i + 1; + } + if (r == 0) + r = jb_put(jb, s->s + from, s->len - from); + if (r == 0) + r = jb_put(jb, "\"", 1); + return r; +} + +struct mget_ctx { + const char *pat; + struct sip_msg *msg; + pv_spec_t *keys_pv, *vals_pv; /* AVP mode */ + struct jbuf *jb; /* JSON mode */ + unsigned int limit, n, now; + int err; +}; + +static int mget_cb(const str *key, const str *val, unsigned int exp, void *p) +{ + struct mget_ctx *mc = p; + pv_value_t pval; + + if (exp && exp <= mc->now) + return 0; /* expired-as-absent */ + if (fnmatch(mc->pat, key->s, 0)) + return 0; + + if (mc->jb) { + if ((mc->n && jb_put(mc->jb, ",", 1) < 0) || + jb_put_jstr(mc->jb, key) < 0 || + jb_put(mc->jb, ":", 1) < 0 || + jb_put_jstr(mc->jb, val) < 0) { + mc->err = 1; + return -1; + } + } else { + memset(&pval, 0, sizeof pval); + pval.flags = PV_VAL_STR; + pval.rs.s = (char *)key->s; + pval.rs.len = key->len; + if (pv_set_value(mc->msg, mc->keys_pv, 0, &pval) < 0) { + mc->err = 1; + return -1; + } + pval.rs.s = (char *)val->s; + pval.rs.len = val->len; + if (pv_set_value(mc->msg, mc->vals_pv, 0, &pval) < 0) { + mc->err = 1; + return -1; + } + } + + mc->n++; + if (mc->limit && mc->n >= mc->limit) + return -1; /* stop: limit reached */ + return 0; +} + +#define PERF_MGET_DEF_LIMIT 1000 + +static int perf_mget_run(struct sip_msg *msg, str *glob, pv_spec_t *keys_pv, + pv_spec_t *vals_pv, struct jbuf *jb, str *col_s, int *limit) +{ + pcache_col_t *col = col_by_name(col_s); + struct mget_ctx mc; + char *pat; + + if (!col) + return -1; + pat = glob_dup(glob); + if (!pat) + return -1; + + memset(&mc, 0, sizeof mc); + mc.pat = pat; + mc.msg = msg; + mc.keys_pv = keys_pv; + mc.vals_pv = vals_pv; + mc.jb = jb; + mc.limit = limit ? (unsigned int)*limit : PERF_MGET_DEF_LIMIT; + mc.now = get_ticks(); + + pcache_ht_iter(col->htable, mget_cb, &mc); + pkg_free(pat); + + return mc.err ? -1 : (int)mc.n; +} + +static int w_perf_mget(struct sip_msg *msg, str *glob, pv_spec_t *keys_pv, + pv_spec_t *vals_pv, str *col_s, int *limit) +{ + int n = perf_mget_run(msg, glob, keys_pv, vals_pv, NULL, col_s, limit); + + return n > 0 ? n : -1; +} + +static int w_perf_mget_json(struct sip_msg *msg, str *glob, pv_spec_t *dst_pv, + str *col_s, int *limit) +{ + struct jbuf jb; + pv_value_t pval; + int n; + + memset(&jb, 0, sizeof jb); + if (jb_put(&jb, "{", 1) < 0) + return -1; + + n = perf_mget_run(msg, glob, NULL, NULL, &jb, col_s, limit); + if (n < 0 || jb_put(&jb, "}", 1) < 0) { + if (jb.s) + pkg_free(jb.s); + return -1; + } + + memset(&pval, 0, sizeof pval); + pval.flags = PV_VAL_STR; + pval.rs.s = jb.s; + pval.rs.len = jb.len; + if (pv_set_value(msg, dst_pv, 0, &pval) < 0) { + pkg_free(jb.s); + return -1; + } + pkg_free(jb.s); + + /* the variable holds "{}" on zero matches; script-false either way */ + return n > 0 ? n : -1; +} + + +/* CP-05 + CP-09: the maintenance timer. Runs in a single timer process + * (so it is the SOLE splitter, which the growth code relies on). First + * reclaims expired records (CP-05, hint-routed - an idle collection costs a + * 16-hints-per-line scan), then grows any collection whose load factor has + * climbed past growth_load_factor (CP-09), bounded per tick. */ +static void pcache_expire_timer(unsigned int ticks, void *param) +{ + pcache_col_t *col; + pcache_ht_totals_t t; + unsigned int now = get_ticks(), freed, split, prev_b, new_b; + + /* one-shot: huge pages were requested but the granted tier is + * sub-optimal. Deferred here from mod_init because EVI has no + * subscribers that early; the shm gate's atomic test-and-set makes it + * fire exactly once even if more than one process runs the timer. */ + if (mem_degraded && mem_degraded_gate && + __sync_bool_compare_and_swap(mem_degraded_gate, 0, 1)) + pcache_raise_degraded(); + + for (col = pcache_collection; col; col = col->next) { + if (!col->htable) + continue; + + /* only pay for the per-key expiry callback where a collection + * opted in AND someone is listening */ + if (col->raise_expired && evi_probe_event(evi_expired_id)) + freed = pcache_ht_sweep(col->htable, now, + pcache_on_expired, &col->col_name); + else + freed = pcache_ht_sweep(col->htable, now, NULL, NULL); + if (freed) + LM_DBG("collection <%.*s>: reclaimed %u expired records\n", + col->col_name.len, col->col_name.s, freed); + + if (growth_load_factor > 0) { + prev_b = pcache_ht_nbuckets(col->htable); + split = pcache_ht_grow(col->htable, + growth_load_factor, growth_budget); + if (split) { + new_b = pcache_ht_nbuckets(col->htable); + LM_DBG("collection <%.*s>: grew by %u splits " + "(%u->%u buckets)\n", col->col_name.len, + col->col_name.s, split, prev_b, new_b); + pcache_ht_totals(col->htable, &t); + pcache_raise_grown(&col->col_name, prev_b, new_b, + split, t.entries); + } + } + } +} + +/* set a per-collection flag for every declared collection named in a CSV + * modparam (event_expired_collections, persist_collections) */ +enum col_flag { COL_FLAG_EXPIRED, COL_FLAG_PERSIST, COL_FLAG_REPLICATE }; +static void mark_collections(char *csv_s, const char *what, enum col_flag f) +{ + csv_record *cr, *c; + pcache_col_t *col; + str csv; + + if (!csv_s || !*csv_s) + return; + csv.s = csv_s; + csv.len = strlen(csv_s); + cr = parse_csv_record(&csv); + for (c = cr; c; c = c->next) { + int found = 0; + for (col = pcache_collection; col; col = col->next) + if (col->col_name.len == c->s.len && + !memcmp(col->col_name.s, c->s.s, c->s.len)) { + if (f == COL_FLAG_EXPIRED) + col->raise_expired = 1; + else if (f == COL_FLAG_REPLICATE) + col->replicate = 1; + else + col->persist = 1; + found = 1; + } + if (!found) + LM_WARN("%s: <%.*s> is not a declared collection\n", + what, c->s.len, c->s.s); + } + free_csv_record(cr); +} + +/* ---- consumer-facing pull API (pull_api.h) ---------------------------- */ + +static int pcache_api_pull_start(cachedb_con *con, str *key, int *fd, + unsigned int *handle) +{ + pcache_col_t *col = pcache_col_of(con); + + if (!col || !key || !fd || !handle) + return -1; + return pcache_pull_start(col, key, 0, fd, handle); +} + +static int pcache_api_pull_start_at(cachedb_con *con, str *key, int node_id, + int *fd, unsigned int *handle) +{ + pcache_col_t *col = pcache_col_of(con); + + if (!col || !key || !fd || !handle) + return -1; + return pcache_pull_start(col, key, node_id, fd, handle); +} + +static int pcache_api_my_node_id(cachedb_con *con) +{ + if (!cluster_ready || !clusterer_api.get_my_id) + return 0; + return clusterer_api.get_my_id(); +} + +static int pcache_api_pull_finish(cachedb_con *con, str *key, + unsigned int handle, str *val) +{ + pcache_col_t *col = pcache_col_of(con); + char buf[PCACHE_PULL_MAX_VAL]; + unsigned int vlen = 0; + int rc; + + if (val) { + val->s = NULL; + val->len = 0; + } + if (!col || !key) + return -1; + + rc = pcache_pull_finish(col, key, handle, buf, sizeof buf, &vlen, NULL); + if (rc != 1 || !val) + return rc; + + /* hand back memory the caller owns, exactly as a get would - the value + * is in the local table too, so a plain get would find it as well */ + val->s = pkg_malloc(vlen ? vlen : 1); + if (!val->s) { + LM_ERR("no more pkg memory for a %u byte pulled value\n", vlen); + return -1; + } + memcpy(val->s, buf, vlen); + val->len = vlen; + return 1; +} + +int load_pcache_pull(pcache_pull_api_t *api) +{ + if (!api) + return -1; + if (!pull_ready) { + LM_WARN("a module asked for the cross-node pull API, but pulling " + "is not configured (replicate_collections)\n"); + return -1; + } + api->start = pcache_api_pull_start; + api->finish = pcache_api_pull_finish; + api->start_at = pcache_api_pull_start_at; + api->my_node_id = pcache_api_my_node_id; + return 0; +} + +static int mod_init(void) +{ + cachedb_engine cde; + cachedb_con *con; + str default_url = str_init("perf://"); + str def_name = str_init(PCACHE_DEFAULT_COLLECTION); + pcache_url_t *it, *next; + pcache_col_t *col; + int i; + + /* which of the four memory backings (DESIGN 2.6.1) does this host + * support? Probed by trying, pre-fork; the arena CONSUMES the + * result only if arena_hugepage_mb>0 (CP-02/CP-20) - with it unset + * (the default), this is a capability check only and every + * cachedb_perf allocation actually goes through plain shm_malloc(), + * fully counted in core's own shmem: stats, not a separate + * reservation. The two NOTICEs below are deliberately worded to + * never be mistaken for each other - a probe result is not a + * report of what is actually in use. */ + pcache_mem_probe(); + + if (pcache_mem.tier == PCACHE_MEM_HUGETLB) + LM_NOTICE("memory backing CAPABILITY PROBE: this host supports " + "tier 1/4 - %s (pool: %d static + %d overcommit pages)\n", + pcache_mem_tier_str(pcache_mem.tier), + pcache_mem.huge_static, pcache_mem.huge_overcommit); + else + LM_NOTICE("memory backing CAPABILITY PROBE: this host supports " + "tier %d/4 - %s\n", + pcache_mem.tier, pcache_mem_tier_str(pcache_mem.tier)); + + if (pcache_arena_hugepage_mb > 0) + LM_NOTICE("memory backing IN USE: a separate %d MB reservation, " + "OUTSIDE OpenSIPS shared memory (arena_hugepage_mb)\n", + pcache_arena_hugepage_mb); + else + LM_NOTICE("memory backing IN USE: OpenSIPS shared memory " + "(shm_malloc) - NOT a separate reservation; counted in core's " + "own shmem: stats, not a cachedb_perf-specific total. Set " + "arena_hugepage_mb to reserve a dedicated arena instead.\n"); + + switch (pcache_mem.tier) { + case PCACHE_MEM_HUGETLB: + break; + case PCACHE_MEM_4K: + LM_WARN("no 2M pages available: on a large cache, pointer-chase " + "reads run up to 1.42x slower because the TLB cannot cover " + "the arena with 4K pages; enable with " + "'sysctl -w vm.nr_overcommit_hugepages=256' (a ceiling of " + "256 x 2M = 512 MB - overcommit pages are taken from free " + "memory only when faulted and returned on exit, so nothing " + "is held while unused)\n"); + break; + default: + LM_WARN("running on THP - most of the TLB win, but hugetlb " + "(tier 1) still measures ~1.2x faster on pointer-chase " + "reads (125 vs 156 ns); enable with " + "'sysctl -w vm.nr_overcommit_hugepages=256' (overcommit " + "pages are taken from free memory only when faulted and " + "returned on exit, so nothing is held while unused)\n"); + } + + /* the slab arena (DESIGN 3.3) - shm globals, pre-fork */ + if (pcache_arena_init() < 0) { + LM_ERR("failed to init the arena\n"); + return -1; + } + + /* CP-11: huge pages were asked for but the arena settled on a lesser + * tier - flagged now, raised from the first timer tick (EVI has no + * subscribers this early) via a shm one-shot gate */ + mem_degraded = (pcache_arena_hugepage_mb > 0 && + pcache_arena_tier() != PCACHE_MEM_HUGETLB); + if (mem_degraded) { + mem_degraded_gate = shm_malloc(sizeof *mem_degraded_gate); + if (!mem_degraded_gate) { + LM_ERR("no more shm memory\n"); + return -1; + } + *mem_degraded_gate = 0; + } + + if (arena_selftest && pcache_arena_selftest() < 0) { + LM_ERR("arena selftest failed\n"); + return -1; + } + + if (htable_selftest && pcache_htable_selftest() < 0) { + LM_ERR("htable selftest failed\n"); + return -1; + } + + memset(&cde, 0, sizeof cde); + cde.name = pcache_mod_name; + + cde.cdb_func.init = pcache_init; + cde.cdb_func.destroy = pcache_destroy; + cde.cdb_func.get = pcache_htable_fetch; + cde.cdb_func.get_buf = pcache_htable_fetch_buf; + cde.cdb_func.get_counter = pcache_htable_fetch_counter; + cde.cdb_func.set = pcache_htable_insert; + cde.cdb_func.remove = pcache_htable_remove; + cde.cdb_func.add = pcache_htable_add; + cde.cdb_func.sub = pcache_htable_sub; + cde.cdb_func.iter_keys = pcache_htable_iter_keys; + + cde.cdb_func.capability = CACHEDB_CAP_BINARY_VALUE | CACHEDB_CAP_GET_BUF; + + if (register_cachedb(&cde) < 0) { + LM_ERR("failed to register the 'perf' cachedb engine\n"); + return -1; + } + + /* CP-11: publish the observability events. A failed publish just + * leaves the id EVI_ERROR and the raise is skipped - never fatal. */ + evi_expired_id = evi_publish_event(evi_expired_name); + evi_nomem_id = evi_publish_event(evi_nomem_name); + evi_grown_id = evi_publish_event(evi_grown_name); + evi_degraded_id = evi_publish_event(evi_degraded_name); + evi_synced_id = evi_publish_event(evi_synced_name); + if (evi_expired_id == EVI_ERROR || evi_nomem_id == EVI_ERROR || + evi_grown_id == EVI_ERROR || evi_degraded_id == EVI_ERROR || + evi_synced_id == EVI_ERROR) + LM_ERR("could not publish one or more cachedb_perf events\n"); + + /* CP-19 Stage 2: cluster sync is a soft, opt-in feature. It needs a DB + * (peers pull from it) and the clusterer module; if either is missing, + * perf_sync degrades to a DB save with no peer signal - never fatal. */ + if (sync_cluster_id > 0) { + if (load_clusterer_api(&clusterer_api) != 0) { + LM_WARN("clusterer module not available - the cluster features " + "are disabled; load clusterer before cachedb_perf\n"); + } else if ((pc_view = shm_malloc(sizeof *pc_view)) == NULL) { + LM_WARN("no shm for the cluster membership view - the cluster " + "features are disabled\n"); + } else if (memset(pc_view, 0, sizeof *pc_view), + clusterer_api.register_capability(&pcache_sync_cap, + pcache_sync_recv, pcache_cluster_event, sync_cluster_id, + 0, NODE_CMP_ANY) < 0) { + LM_WARN("could not register the cluster capability - the " + "cluster features are disabled\n"); + } else { + cluster_ready = 1; + /* the DB is what perf_sync snapshots through; a cache that + * only pulls has no use for one */ + if (db_url && *db_url) { + sync_ready = 1; + LM_INFO("cluster sync active on cluster_id %d (cap <%.*s>)\n", + sync_cluster_id, pcache_sync_cap.len, pcache_sync_cap.s); + } else { + LM_INFO("cluster membership active on cluster_id %d; " + "perf_sync needs db_url and stays disabled\n", + sync_cluster_id); + } + } + } + + /* CP-15.12: arm the failover sync on a sharing tag. Independent of + * sync_cluster_id (a deployment may want only the failover hook), so + * bind the clusterer API here if the sync block did not. */ + if (sync_shtag_str && *sync_shtag_str) { + char *slash = strchr(sync_shtag_str, '/'); + + pc_shtag.s = sync_shtag_str; + pc_shtag.len = slash ? (int)(slash - sync_shtag_str) + : (int)strlen(sync_shtag_str); + pc_shtag_cid = slash ? atoi(slash + 1) : sync_cluster_id; + + if (!pc_shtag.len || pc_shtag_cid <= 0) { + LM_WARN("bad sync_shtag '%s' (expected \"name/cluster_id\") - " + "failover sync disabled\n", sync_shtag_str); + } else if (!(db_url && *db_url)) { + LM_WARN("sync_shtag is set but db_url is not - the failover " + "sync needs the DB snapshot; disabled\n"); + } else if (!cluster_ready && load_clusterer_api(&clusterer_api) != 0) { + LM_WARN("clusterer module not available - failover sync " + "disabled\n"); + } else if (clusterer_api.shtag_register_callback(&pc_shtag, + pc_shtag_cid, NULL, pcache_shtag_cb) < 0) { + LM_WARN("cannot register on sharing tag <%.*s/%d> - failover " + "sync disabled\n", pc_shtag.len, pc_shtag.s, pc_shtag_cid); + } else { + LM_INFO("failover sync armed on sharing tag <%.*s/%d>\n", + pc_shtag.len, pc_shtag.s, pc_shtag_cid); + } + } + + /* CP-15.5: cross-node pull. Opt-in per collection, and inert without + * it: a key is only worth asking the cluster about if it means the + * same thing on every node, which only the operator knows. */ + if (replicate_collections && *replicate_collections) { + int use_clctr = pull_transport_str && + !strcasecmp(pull_transport_str, "clctr"); + + if (pull_transport_str && strcasecmp(pull_transport_str, "bin") && + !use_clctr) { + LM_ERR("bad pull_transport '%s' - expected 'bin' or 'clctr'\n", + pull_transport_str); + return -1; + } + if (use_clctr) { + /* The controller is optional at build time AND at run time. + * An explicit clctr choice this deployment cannot honour + * degrades to the bin transport - or to no pull at all if + * the clusterer is missing too, which the cluster_ready + * check below already handles. Loudly, but the cache + * itself is never held hostage by its cluster plane. */ +#ifdef CLUSTERER_CTRL_SUPPORT + if (load_clctr_api(&clctr_api) < 0) { + LM_WARN("pull_transport 'clctr' but clusterer_controller " + "is not loaded - falling back to 'bin'\n"); + } else if (clctr_api.register_channel(&pull_channel, + pcache_clctr_recv) < 0) { + LM_WARN("cannot register the pull channel with " + "clusterer_controller - falling back to 'bin'\n"); + } else { + pull_via_clctr = 1; + } +#else + LM_WARN("pull_transport 'clctr' but this build carries no " + "clusterer_controller support - falling back to 'bin'\n"); +#endif + } + if (!cluster_ready) { + LM_WARN("replicate_collections is set but the cluster is not " + "available (needs sync_cluster_id + clusterer) - cross-node " + "pull disabled\n"); + } else if (pull_timeout_ms <= 0 || pull_timeout_ms > 5000) { + LM_ERR("pull_timeout_ms must be within 1..5000\n"); + return -1; + } else { + if (pull_max_value < 1 || pull_max_value > PCACHE_PULL_MAX_VAL) { + LM_WARN("pull_max_value %d out of range 1..%d - clamping\n", + pull_max_value, PCACHE_PULL_MAX_VAL); + pull_max_value = pull_max_value < 1 + ? PCACHE_PULL_MAX_VAL_DEF : PCACHE_PULL_MAX_VAL; + } + if (pull_max_key < 1 || pull_max_key > PCACHE_PULL_MAX_KEY) { + LM_WARN("pull_max_key %d out of range 1..%d - clamping\n", + pull_max_key, PCACHE_PULL_MAX_KEY); + pull_max_key = pull_max_key < 1 + ? PCACHE_PULL_MAX_KEY_DEF : PCACHE_PULL_MAX_KEY; + } + pull_slot_sz = (int)sizeof(struct pcache_pull_slot) + + pull_max_key + pull_max_value; + LM_INFO("cross-node pull: %d slots x %d bytes " + "(key %d, value %d) = %d KB of shm\n", + PCACHE_PULL_SLOTS, pull_slot_sz, pull_max_key, + pull_max_value, + (PCACHE_PULL_SLOTS * pull_slot_sz + 1023) / 1024); + pull_slots = shm_malloc((size_t)PCACHE_PULL_SLOTS * pull_slot_sz); + pull_next_id = shm_malloc(sizeof *pull_next_id); + pull_stats = shm_malloc(PULL_ST_MAX * sizeof *pull_stats); + pull_send_warn = shm_malloc(sizeof *pull_send_warn); + pull_xcluster_warn = shm_malloc(sizeof *pull_xcluster_warn); + peer_stats = shm_malloc((CL_MAX_NODE_ID + 1) * sizeof *peer_stats); + pull_lock = lock_alloc(); + if (!pull_slots || !pull_next_id || !pull_stats || !peer_stats || + !pull_send_warn || !pull_xcluster_warn || + !pull_lock || !lock_init(pull_lock)) { + LM_ERR("no shm for the cross-node pull state\n"); + return -1; + } + memset(pull_slots, 0, (size_t)PCACHE_PULL_SLOTS * pull_slot_sz); + memset(peer_stats, 0, + (CL_MAX_NODE_ID + 1) * sizeof *peer_stats); + /* One eventfd per slot, created HERE - before the fork - so + * that every worker inherits every fd. This is the whole + * reason the pool is fixed and preallocated: a reply arrives + * in whichever process the transport chose, and it has to be + * able to wake the process that asked. An fd created after + * the fork exists only in its own process and could not. */ + for (i = 0; i < PCACHE_PULL_SLOTS; i++) { + pull_slot_at(i)->efd = eventfd(0, EFD_NONBLOCK); + if (pull_slot_at(i)->efd < 0) { + LM_ERR("cannot create the pull wakeup fds: %s\n", + strerror(errno)); + return -1; + } + } + *pull_next_id = 0; + memset(pull_stats, 0, PULL_ST_MAX * sizeof *pull_stats); + memset(pull_send_warn, 0, sizeof *pull_send_warn); + memset(pull_xcluster_warn, 0, sizeof *pull_xcluster_warn); + if (pull_negative_ms < 0 || pull_negative_ms > 2000) { + LM_ERR("pull_negative_ms must be within 0..2000 (0 = off) " + "- a negative that outlives a retransmit turns a " + "transient miss into a hard failure\n"); + return -1; + } + if (pull_negative_ms > 0) { + neg_slots = shm_malloc(PCACHE_NEG_SLOTS * sizeof *neg_slots); + neg_lock = lock_alloc(); + if (!neg_slots || !neg_lock || !lock_init(neg_lock)) { + LM_ERR("no shm for the negative cache\n"); + return -1; + } + memset(neg_slots, 0, PCACHE_NEG_SLOTS * sizeof *neg_slots); + } + mark_collections(replicate_collections, "replicate_collections", + COL_FLAG_REPLICATE); + /* Reclaim slots whose answer never became conclusive. A + * microsecond timer rather than the second-grained expiry + * sweep: pull_timeout_ms is set in milliseconds and a + * suspended lookup should not wait whole seconds past it. + * Checked at half the timeout so a slot is reclaimed within + * ~1.5x of it, and never tied to expiry_sweep_period, which + * an operator is allowed to switch off entirely. */ + { + unsigned int iv = (unsigned int)pull_timeout_ms * 1000 / 2; + + if (iv < 10000) + iv = 10000; /* no tighter than 10 ms */ + if (register_utimer("cachedb-perf-pull-reap", + pcache_pull_reap, NULL, iv, + TIMER_FLAG_DELAY_ON_DELAY) < 0) { + LM_ERR("failed to register the pull reaper - a pull " + "that never gets a conclusive answer would hold " + "its slot for ever\n"); + return -1; + } + } + pull_ready = 1; + /* One pair of stats per collection, named - + * via build_stat_name() (the same convention call_center uses + * for its per-flow stats). Registered here rather than in the + * static table because the collection list is only known after + * cache_collections has been parsed. A failure is not fatal: + * losing a statistic must never stop the module serving + * traffic, so it warns and carries on. */ + { + pcache_col_t *sc; + + for (sc = pcache_collection; sc; sc = sc->next) { + char *nm; + + if (!sc->replicate) + continue; /* cannot be pulled, so always 0 */ + nm = pcache_stat_name(sc, "pulled_from_cluster"); + if (!nm || register_stat2("cachedb_perf", nm, + (stat_var **)smf_col_pulled_in, + STAT_SHM_NAME|STAT_IS_FUNC, (void *)sc, 0) != 0) + LM_WARN("could not register the pulled_from_cluster " + "statistic for collection <%.*s>\n", + sc->col_name.len, sc->col_name.s); + nm = pcache_stat_name(sc, "served_to_cluster"); + if (!nm || register_stat2("cachedb_perf", nm, + (stat_var **)smf_col_served_out, + STAT_SHM_NAME|STAT_IS_FUNC, (void *)sc, 0) != 0) + LM_WARN("could not register the served_to_cluster " + "statistic for collection <%.*s>\n", + sc->col_name.len, sc->col_name.s); + } + } + LM_INFO("cross-node pull active over %s, %d ms timeout, " + "%d ms negative cache, collections: %s\n", + pull_via_clctr ? "clusterer_controller multicast" : "bin", + pull_timeout_ms, pull_negative_ms, replicate_collections); + if (pull_on_miss) + LM_WARN("pull_on_miss is enabled: a cache miss now BLOCKS " + "the calling process for up to %d ms while the cluster " + "is asked. That is fine for a maintenance or test " + "path; on a SIP path it costs a worker, so keep it off " + "until the lookup can be suspended instead\n", + pull_timeout_ms); + } + } + + /* make sure the default collection exists */ + for (col = pcache_collection; col; col = col->next) + if (col->col_name.len == def_name.len && + !memcmp(col->col_name.s, def_name.s, def_name.len)) + break; + + if (!col) { + col = shm_malloc(sizeof *col); + if (!col) { + LM_ERR("no more shm memory\n"); + return -1; + } + memset(col, 0, sizeof *col); + + if (shm_str_dup(&col->col_name, &def_name) < 0) { + LM_ERR("no more shm memory\n"); + shm_free(col); + return -1; + } + col->size_log2 = PCACHE_SIZE_DEFAULT; + + col->next = pcache_collection; + pcache_collection = col; + } + + /* one table per collection, pre-fork */ + for (col = pcache_collection; col; col = col->next) { + col->htable = pcache_htable_new(col->size_log2); + if (!col->htable) { + LM_ERR("failed to create the table for collection <%.*s>\n", + col->col_name.len, col->col_name.s); + return -1; + } + LM_DBG("collection <%.*s>: 2^%u buckets\n", + col->col_name.len, col->col_name.s, col->size_log2); + } + + /* CP-11 / CP-19: per-collection opt-ins */ + mark_collections(event_expired_collections, "event_expired_collections", + COL_FLAG_EXPIRED); + mark_collections(persist_collections, "persist_collections", + COL_FLAG_PERSIST); + + /* CP-19: bind the DB backend and load the persisted collections before + * the workers fork (so every worker starts with a warm cache) */ + if (db_url && *db_url) { + str url = { db_url, strlen(db_url) }; + str tbl = { db_table, strlen(db_table) }; + + if (pcache_db_init(&url, &tbl) < 0) + return -1; + if (db_mode >= 1) + for (col = pcache_collection; col; col = col->next) + if (col->persist && col->htable) + pcache_db_load(col); + } else if (db_mode) { + LM_WARN("db_mode is set but db_url is not - persistence disabled\n"); + } + + /* one script connection per configured URL, or a default one */ + if (pcache_url_list) { + for (it = pcache_url_list; it; it = next) { + next = it->next; + + con = pcache_init(&it->url); + if (!con) { + LM_ERR("failed to init connection for URL <%.*s>\n", + it->url.len, it->url.s); + return -1; + } + + if (cachedb_put_connection(&pcache_mod_name, con) < 0) { + LM_ERR("failed to register connection for URL <%.*s>\n", + it->url.len, it->url.s); + return -1; + } + + /* a groupless URL becomes the engine's default connection; + * remember its collection for the glob functions */ + if (!((pcache_con *)con->data)->id->group_name) + pcache_default_col = ((pcache_con *)con->data)->col; + + pkg_free(it); + } + pcache_url_list = NULL; + } else { + con = pcache_init(&default_url); + if (!con) { + LM_ERR("failed to init the default connection\n"); + return -1; + } + + if (cachedb_put_connection(&pcache_mod_name, con) < 0) { + LM_ERR("failed to register the default connection\n"); + return -1; + } + + pcache_default_col = ((pcache_con *)con->data)->col; + } + + if (expiry_sweep_period > 0) { + if (register_timer("cachedb-perf-expire", pcache_expire_timer, + NULL, expiry_sweep_period, TIMER_FLAG_DELAY_ON_DELAY) < 0) { + LM_ERR("failed to register the expiry sweep timer\n"); + return -1; + } + } else { + LM_WARN("expiry sweep disabled: expired records stay invisible " + "but their memory is never reclaimed\n"); + } + + return 0; +} + +static int child_init(int rank) +{ + /* drop any allocator state inherited over fork - two processes must + * never share a bump pointer (pcache_arena.h) */ + pcache_arena_child_init(); + return 0; +} + +static void mod_destroy(void) +{ + pcache_col_t *col, *next; + + /* CP-19: persist the marked collections on a graceful shutdown */ + if (db_mode >= 2 && pcache_db_enabled()) + for (col = pcache_collection; col; col = col->next) + if (col->persist && col->htable) + pcache_db_save(col); + + for (col = pcache_collection; col; col = next) { + next = col->next; + if (col->col_name.s) + shm_free(col->col_name.s); + shm_free(col); + } + pcache_collection = NULL; + + if (mem_degraded_gate) { + shm_free(mem_degraded_gate); + mem_degraded_gate = NULL; + } + + if (pull_slots) { + int i; + + for (i = 0; i < PCACHE_PULL_SLOTS; i++) + if (pull_slot_at(i)->efd >= 0) + close(pull_slot_at(i)->efd); + shm_free(pull_slots); + pull_slots = NULL; + } + if (pull_lock) { + lock_destroy(pull_lock); + lock_dealloc(pull_lock); + pull_lock = NULL; + } + if (neg_slots) { + shm_free(neg_slots); + neg_slots = NULL; + } + if (neg_lock) { + lock_destroy(neg_lock); + lock_dealloc(neg_lock); + neg_lock = NULL; + } + if (pull_next_id) { + shm_free(pull_next_id); + pull_next_id = NULL; + } + if (pull_stats) { + shm_free(pull_stats); + pull_stats = NULL; + } + if (pull_send_warn) { + shm_free(pull_send_warn); + pull_send_warn = NULL; + } + if (peer_stats) { + shm_free(peer_stats); + peer_stats = NULL; + } + + pcache_arena_destroy(); +} + + +/* + * "name1=S;name2" - S is the log2 of the initial bucket count, clamped + * to [PCACHE_SIZE_MIN, PCACHE_SIZE_MAX], PCACHE_SIZE_DEFAULT if absent + */ +static int pcache_parse_collections(unsigned int type, void *val) +{ + str collection_list, name; + unsigned int size_log2; + pcache_col_t *new_col, *dup; + csv_record *cols, *col, *kv = NULL; + + if (!val) { + LM_ERR("null 'cache_collections' value\n"); + return -1; + } + + init_str(&collection_list, (char *)val); + cols = __parse_csv_record(&collection_list, 0, ';'); + if (!cols) { + LM_ERR("failed to parse 'cache_collections'\n"); + return -1; + } + + for (col = cols; col; col = col->next) { + kv = __parse_csv_record(&col->s, 0, '='); + if (!kv) + goto error; + name = kv->s; + + if (ZSTR(name)) { + LM_DBG("skipping empty collection name\n"); + free_csv_record(kv); + kv = NULL; + continue; + } + + if (name.len >= 2 && name.s[name.len-2] == '/' + && name.s[name.len-1] == 'r') { + LM_ERR("collection <%.*s>: replication ('/r') is not " + "supported, cachedb_perf is a single-node cache\n", + name.len, name.s); + goto error; + } + + if (kv->next) { + if (str2int(&kv->next->s, &size_log2) < 0) { + LM_ERR("collection <%.*s>: invalid size <%.*s>, " + "expected a power-of-2 exponent\n", + name.len, name.s, + kv->next->s.len, kv->next->s.s); + goto error; + } + + if (size_log2 < PCACHE_SIZE_MIN) { + LM_WARN("collection <%.*s>: size %u below minimum, " + "clamping to %u\n", name.len, name.s, + size_log2, PCACHE_SIZE_MIN); + size_log2 = PCACHE_SIZE_MIN; + } else if (size_log2 > PCACHE_SIZE_MAX) { + LM_WARN("collection <%.*s>: size %u above maximum, " + "clamping to %u\n", name.len, name.s, + size_log2, PCACHE_SIZE_MAX); + size_log2 = PCACHE_SIZE_MAX; + } + } else { + size_log2 = PCACHE_SIZE_DEFAULT; + } + + for (dup = pcache_collection; dup; dup = dup->next) { + if (!str_strcmp(&name, &dup->col_name)) { + LM_ERR("collection <%.*s> defined more than once\n", + name.len, name.s); + goto error; + } + } + + new_col = shm_malloc(sizeof *new_col); + if (!new_col) { + LM_ERR("no more shm memory\n"); + goto error; + } + memset(new_col, 0, sizeof *new_col); + + if (shm_str_dup(&new_col->col_name, &name) < 0) { + LM_ERR("no more shm memory\n"); + shm_free(new_col); + goto error; + } + new_col->size_log2 = size_log2; + + add_last(new_col, pcache_collection); + + LM_DBG("collection <%.*s>, initial size 2^%u buckets\n", + name.len, name.s, size_log2); + + free_csv_record(kv); + kv = NULL; + } + + free_csv_record(cols); + return 0; + +error: + LM_ERR("failed to parse 'cache_collections'\n"); + if (kv) + free_csv_record(kv); + free_csv_record(cols); + return -1; +} + + +/* URLs are stored until mod_init, when all collections are known */ +static int pcache_store_urls(unsigned int type, void *val) +{ + pcache_url_t *new_url; + + new_url = pkg_malloc(sizeof *new_url); + if (!new_url) { + LM_ERR("no more pkg memory\n"); + return -1; + } + + init_str(&new_url->url, (char *)val); + new_url->next = pcache_url_list; + pcache_url_list = new_url; + + return 0; +} diff --git a/modules/cachedb_perf/cachedb_perf.h b/modules/cachedb_perf/cachedb_perf.h new file mode 100644 index 00000000000..eef6f73f8f1 --- /dev/null +++ b/modules/cachedb_perf/cachedb_perf.h @@ -0,0 +1,81 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _CACHEDB_PERF_H_ +#define _CACHEDB_PERF_H_ + +#include "../../cachedb/cachedb.h" +#include "../../cachedb/cachedb_cap.h" + +/* log2 of a collection's initial bucket count. Growth (CP-09) resizes at + * runtime, so this only sets the starting point. Configured values are + * clamped to [PCACHE_SIZE_MIN, PCACHE_SIZE_MAX]: an unbounded "1 << size" + * is undefined behaviour at 32 and a zero-size table at 64. */ +#define PCACHE_SIZE_MIN 4 +#define PCACHE_SIZE_MAX 24 +#define PCACHE_SIZE_DEFAULT 14 + +#define PCACHE_DEFAULT_COLLECTION "default" + +struct pcache_htable; + +typedef struct pcache_col { + str col_name; + unsigned int size_log2; /* initial table size (log2 buckets) */ + struct pcache_htable *htable; + int raise_expired; /* CP-11: emit E_CACHEDB_PERF_EXPIRED */ + int persist; /* CP-19: load-on-start / save-on-stop */ + int replicate; /* CP-15: may be pulled across nodes */ + /* cluster-sync observability (shm: written by whichever process runs + * the sync, read by perf_stats). These record when this node last + * pushed or pulled - NOT that the caches currently match. */ + unsigned int last_sync_out; /* ticks: last save-and-broadcast here */ + unsigned int last_sync_in; /* ticks: last reload asked for by a peer */ + int last_sync_src; /* node id that asked for that reload */ + /* CP-15 convergence, PER COLLECTION. The pull_stats[] counters are + * module-wide, so with more than one collection they cannot show WHICH + * one is converging - and last_sync_out/in stay -1 forever unless + * perf_sync is explicitly invoked, which says nothing about pull-based + * convergence. These two do. shm, written by any worker, so both are + * touched only with __sync_fetch_and_add. */ + unsigned long pulled_in; /* values fetched from a peer AND stored */ + unsigned long served_out; /* times we answered a peer WITH a value */ + struct pcache_col *next; +} pcache_col_t; + +/* the first 3 fields must mirror cachedb_pool_con (cachedb/cachedb_pool.h) */ +typedef struct { + struct cachedb_id *id; + unsigned int ref; + struct cachedb_pool_con_t *next; + + pcache_col_t *col; +} pcache_con; + +typedef struct pcache_url { + str url; + struct pcache_url *next; +} pcache_url_t; + +extern pcache_col_t *pcache_collection; + +#endif /* _CACHEDB_PERF_H_ */ diff --git a/modules/cachedb_perf/doc/cachedb_perf.xml b/modules/cachedb_perf/doc/cachedb_perf.xml new file mode 100644 index 00000000000..edc6c491379 --- /dev/null +++ b/modules/cachedb_perf/doc/cachedb_perf.xml @@ -0,0 +1,24 @@ + + + + + +%docentities; + +]> + + + + cachedb_perf Module + &osipsname; + + + + &admin; + + &docCopyrights; + ©right; 2026 Yury Kirsanov + diff --git a/modules/cachedb_perf/doc/cachedb_perf_admin.xml b/modules/cachedb_perf/doc/cachedb_perf_admin.xml new file mode 100644 index 00000000000..3741923f660 --- /dev/null +++ b/modules/cachedb_perf/doc/cachedb_perf_admin.xml @@ -0,0 +1,1221 @@ + + + + + &adminguide; + +
+ Overview + + This module is a high-performance local memory cache implementing + the Key-Value interface exported by the OpenSIPS core. It is a + drop-in alternative to cachedb_local, selected + by URL scheme (perf:// instead of + local://), designed for large, high-churn + caches: lock-free reads, cache-line-sized buckets and a table that + grows at runtime instead of being sized once at startup. + + + Each OpenSIPS instance keeps its own in-memory copy, but a whole + collection can be persisted to an SQL backend so it survives a + restart (see ), and refreshed + cluster-wide from that shared DB with perf_sync + (see ). What the module does + not do is cachedb_local-style + per-operation replication (a cluster_id write + streamed between nodes on every operation) - that would tax the + lock-free path the module exists to keep fast; sharing here is the + pull-from-DB refresh model. Deployments that need per-operation + replication must stay on cachedb_local. + + + Data is organized in named collections (hash + tables), declared via the cache_collections + parameter. Each cachedb_url points to one + collection; a URL naming no collection uses the collection named + default, which always exists. + + + + Work in progress: this module is under active development, but + functionally complete for a single node - data operations, the + background expiry sweep, runtime table growth, statistics, the + huge-page arena, the introspection MI and the observability events + are all in place, whole collections can be persisted to a db_* + backend, and perf_sync refreshes a collection cluster-wide from that + shared DB. Per-operation replication (a streamed write log between + nodes) is intentionally out of scope - sharing is the pull-from-DB + refresh model, not a merge of divergent copies. + + +
+ +
+ Dependencies +
+ &osips; Modules + + None required. Two are optional: + + + + clusterer - enables the cluster features: + cross-node pull over its BIN links, the perf_sync peer signal and + the sync_shtag failover hook. Without it the module runs purely + node-local (see ). + + + clusterer_controller - enables the optional + clctr pull transport (encrypted multicast). + Optional at build time too: a tree built without it simply has no + clctr transport, and everything runs over the clusterer's BIN + links. + + +
+
+ External Libraries or Applications + + None. + +
+
+ +
+ Exported Parameters + +
+ <varname>cache_collections</varname> (string) + + Declares the collections and, optionally, their initial hash + table size, as a semicolon-separated list of + name or name=size + entries. The size is the power-of-2 exponent of the initial + bucket count (as in cachedb_local) and only + sets the starting point - the table grows at runtime as entries + accumulate. Values are clamped to the [4, 24] range; the default + is 14 (16384 buckets). + + + The cachedb_local replication marker + (/r) is rejected: this cache is single-node. + + + Set <varname>cache_collections</varname> parameter + +... +modparam("cachedb_perf", "cache_collections", "th=16;profiles") +... + + +
+ +
+ <varname>expiry_sweep_period</varname> (integer) + + How often, in seconds, expired records are reclaimed. Expired + entries are already invisible to reads the moment they expire - + the sweep only frees their memory, guided by per-bucket hints so + idle collections cost next to nothing. Default is 1 second; 0 + disables the sweep (expired records then hold their memory until + overwritten or deleted). + + + Set <varname>expiry_sweep_period</varname> parameter + +... +modparam("cachedb_perf", "expiry_sweep_period", 5) +... + + +
+ +
+ <varname>cachedb_url</varname> (string) + + URL(s) usable from the script or by other modules. The collection + is given by the URL's database part + (perf:///name) or, equivalently, its host + part (perf://name) - a host has no meaning + for a local cache, so both forms select the collection. A URL + naming no collection (perf://) uses the + default collection. Naming an undefined collection + is a startup error. Multiple URLs may share one collection; use a + group (perf:group_name:///name) to address a + specific URL from the script. + + + Set <varname>cachedb_url</varname> parameter + +... +modparam("cachedb_perf", "cachedb_url", "perf:///th") +modparam("cachedb_perf", "cachedb_url", "perf:prof:///profiles") + +# usage from script: +# cache_store("perf", ...) - collection "th" +# cache_store("perf:prof", ...) - collection "profiles" +... + + +
+ +
+ <varname>growth_load_factor</varname> (integer) + + The target number of entries per bucket the maintenance timer grows + the table toward. As entries accumulate the timer splits buckets to + keep the load factor near this value, so lookups stay flat as the + cache scales - the behaviour cachedb_local + lacks. 0 disables growth, leaving the table fixed at its declared + size. Default is 2. + + + Set <varname>growth_load_factor</varname> parameter + +... +modparam("cachedb_perf", "growth_load_factor", 2) +... + + +
+ +
+ <varname>growth_budget</varname> (integer) + + The maximum number of bucket splits the maintenance timer performs + on a single run, bounding the work of one growth pass so the timer + never stalls under a burst of inserts. Default is 4096. + + + Set <varname>growth_budget</varname> parameter + +... +modparam("cachedb_perf", "growth_budget", 4096) +... + + +
+ +
+ <varname>arena_hugepage_mb</varname> (integer) + + Size, in megabytes, of a huge-page-backed reservation for the cache + entries. When set, the module reserves this much memory at startup + and backs it with 2 MB pages to cut TLB misses on a large cache. It + does not pick a mechanism: it climbs a detect-by-trying ladder - + overcommit hugetlb pool (MAP_HUGETLB) then + transparent huge pages (MADV_HUGEPAGE) then + MADV_COLLAPSE then plain 4 KB - and keeps the + best tier the running kernel actually grants, which it reports at + startup and through the memory_tier statistic + and perf_stats. 0 (default) uses plain + demand-faulted shared memory. + + + To make the faster tiers available: allow on-demand huge pages with + sysctl vm.nr_overcommit_hugepages=N (N >= + arena_hugepage_mb/2), and/or enable shmem THP + with echo advise > /sys/kernel/mm/transparent_hugepage/shmem_enabled. + Except for the hugetlb tier (which is unswappable and exempt), the + reservation is mlock-pinned against swap; that + needs LimitMEMLOCK=infinity in the systemd unit, + otherwise the module warns and runs the arena unpinned. + + + Set <varname>arena_hugepage_mb</varname> parameter + +... +modparam("cachedb_perf", "arena_hugepage_mb", 512) +... + + +
+ +
+ <varname>arena_selftest</varname> (integer) + + When set to 1, the slab arena runs a self-test at startup and aborts + startup on any mismatch - a permanent, cheap diagnostic. Default is 0 + (off). + + + Set <varname>arena_selftest</varname> parameter + +... +modparam("cachedb_perf", "arena_selftest", 1) +... + + +
+ +
+ <varname>htable_selftest</varname> (integer) + + When set to 1, the hash table and its runtime-growth machinery run a + self-test at startup and abort startup on any mismatch. Default is 0 + (off). + + + Set <varname>htable_selftest</varname> parameter + +... +modparam("cachedb_perf", "htable_selftest", 1) +... + + +
+ +
+ <varname>event_expired_collections</varname> (string) + + Comma-separated list of the collections that raise + E_CACHEDB_PERF_EXPIRED (one event per reaped key) + as the sweep reclaims them. It is opt-in per collection because a + high-churn collection can reap in bulk, and event delivery is + synchronous - a collection should pay for the per-key events only if + something is listening for them. Empty (default) means no collection + raises the event. See . + + + Set <varname>event_expired_collections</varname> parameter + +... +modparam("cachedb_perf", "event_expired_collections", "sessions,subscriptions") +... + + +
+ +
+ <varname>db_url</varname> (string) + + URL of a db_* (SQL) backend used to persist + collections - see . The matching + db_* module must be loaded. When unset, + persistence is disabled. The DB is a shared, durable store; the + in-memory cache is a view over it. + + + Set <varname>db_url</varname> parameter + +... +modparam("cachedb_perf", "db_url", "mysql://opensips:pw@localhost/opensips") +... + + +
+ +
+ <varname>db_table</varname> (string) + + Table that holds the persisted entries. Default is + cachedb_perf. See for + the schema. + +
+ +
+ <varname>db_mode</varname> (integer) + + Automatic persistence for the collections listed in + : 0 = off (default; + load/save only on the perf_load/ + perf_save MI commands), 1 = load them from the + DB at startup, 2 = load at startup and save on a graceful shutdown. + + + Set <varname>db_mode</varname> parameter + +... +modparam("cachedb_perf", "db_mode", 2) +... + + +
+ +
+ <varname>persist_collections</varname> (string) + + Comma-separated list of the collections that + loads at startup and saves at + shutdown. Empty (default) means none are persisted automatically - + though perf_save/perf_load + still work on any collection on demand. + + + Set <varname>persist_collections</varname> parameter + +... +modparam("cachedb_perf", "persist_collections", "sessions,profiles") +... + + +
+ +
+ <varname>sync_cluster_id</varname> (integer) + + Cluster to signal on perf_sync - see + . 0 (default) = off. When set, the + clusterer module must be loaded (before + cachedb_perf) and + configured; if either is missing, perf_sync + degrades to a DB save with no peer signal (a soft dependency, never + fatal). + + + Set <varname>sync_cluster_id</varname> parameter + +... +loadmodule "clusterer.so" +loadmodule "cachedb_perf.so" +modparam("cachedb_perf", "sync_cluster_id", 1) +... + + +
+
+ <varname>sync_shtag</varname> (string) + + A clusterer sharing tag, as name/cluster_id, that + arms the failover sync. A node whose tag turns + active warms every declared collection from + the DB snapshot before the redirected traffic arrives; a node + gracefully demoted to backup saves its + state and signals the peers to reload it - so a failover moves + the cache as one snapshot instead of a storm of misses. + + + Requires db_url. The tag only schedules + these bulk operations - lookups are never gated on its state. + On a crash failover the last saved snapshot is the only source, + so pair this with periodic perf_save (or + db_mode 2) on the active node. + + + + Default value is unset (failover sync off). + + + + Setting <varname>sync_shtag</varname> + +modparam("clusterer", "sharing_tag", "vip1/1=backup") +modparam("cachedb_perf", "sync_shtag", "vip1/1") + + +
+
+ <varname>replicate_collections</varname> (string) + + Comma-separated collections whose keys may be fetched from + another node when this one misses (pull on miss). + Nothing is pulled unless it is listed here, and the default is + to list nothing. + + + The opt-in is deliberate and cannot be inferred: a key is only + worth asking the cluster about if it means the same thing on + every node. That holds for keys derived from the call - the + topology hiding state, for instance - and fails for anything a + script names after something local, where a peer's answer would + be wrong rather than merely useless. Values must be portable + too: a blob that embeds a node's own address is not. + + + Requires sync_cluster_id and the clusterer. + + + A pulled key is kept: the next request for it + is answered locally and the cluster is never asked again, which is + what makes this a repair rather than a relay. The copy keeps the + expiry the owner had - never a fresh lifetime - so it dies when + the original does instead of outliving it. Note the consequence + for sizing: as traffic spreads, every node tends toward holding + every key, so size the arena for the whole keyspace rather than + its share of it. + + + Native counters (created with cache_add) are + never served to a peer. A counter records what happened on the + node holding it, so handing it over would import one node's tally + into another; the requester is told the key is not there, which + from its side is true. + + + + Default value is unset (no collection is pulled). + + + + Setting <varname>replicate_collections</varname> + +modparam("cachedb_perf", "sync_cluster_id", 1) +modparam("cachedb_perf", "replicate_collections", "th") + + +
+ +
+ <varname>pull_transport</varname> (string) + + How cross-node pulls travel: bin (default) + uses the clusterer's BIN links. clctr rides + the clusterer_controller module's encrypted multicast plane + instead: one datagram reaches every peer, and the payload is + encrypted, which the BIN links are not. + + + The controller is optional, at build time and at run time. When + this build does not include clusterer_controller, or the module + is not loaded, clctr logs a warning and + degrades to bin; if the clusterer module is + unavailable too, cross-node pull and sync are disabled and the + cache runs purely node-local. Missing cluster infrastructure + never stops the module from starting. + + + + Default value is bin. + + +
+ +
+ <varname>pull_timeout_ms</varname> (integer) + + How long a pull waits for the cluster, in milliseconds (1..5000). + It is a backstop, not the normal cost: with every peer answering + either way, a pull finishes as soon as the last one has spoken - + on a LAN, in a couple of milliseconds. The timeout only decides + how long an unanswered request lingers. + + + + Default value is 50. + + +
+
+ <varname>pull_linger_ms</varname> (integer) + + A pull whose answer arrives after + pull_timeout_ms is no longer anyone's answer - + the caller has moved on - but it is still a perfectly good value. + The module keeps the request slot around and stores such a late + answer into the cache, so the next lookup hits locally instead of + asking the cluster again. Without this, the cache never converges + on exactly the keys that are hardest to fetch. + + + This parameter is an optional extra bound on + how late is too late, in milliseconds past the pull timeout. The + default of 0 means no time bound: what makes a late + answer valid is that the value is still + valid, and the value carries its own expiry, computed against + this node's clock. The practical ceiling is the request slot's + own lifetime. + + + Set it non-zero only where the script deletes + keys from a replicated collection: a late store cannot tell + never had it from deleted a moment + ago, so a peer's copy could resurrect a key the script + removed. A deployment that only writes and lets TTLs expire + cannot hit that. A late answer never overwrites a live local + entry in any case - it fills gaps, it does not compete with + fresher writes. + + + + Default value is 0 (no extra bound). + + +
+
+ <varname>pull_negative_ms</varname> (integer) + + How long to remember that the whole cluster answered not + here for a key, in milliseconds (0..2000; 0 disables it). + A SIP retransmit asks the same question a few hundred + milliseconds later, and without this every retransmit repeats the + round of questions. + + + Keep it short. A key may legitimately be created on another node + a moment from now, and a negative that outlives that turns a + transient miss into a hard failure - which is why the parameter + is capped rather than left open. Only a verdict the whole + cluster gave is remembered: a timeout is not absence, and neither + is an answer from a set of nodes that has since changed. A local + write to the key clears it at once. + + + Negatives are held outside the cache, so they never appear in + perf_keys or perf_dump + and never count as entries. + + + + Default value is 300. + + +
+
+ <varname>pull_on_miss</varname> (integer) + + Repair a miss on the ordinary read path: when a lookup finds + nothing locally, ask the cluster and return whatever comes back + as though it had been here all along. A consumer needs no + changes - cross-node lookups simply start working for the + collections listed in + . + + + Off by default, and it should stay off on a + SIP path for now. The lookup blocks until the cluster + answers or pull_timeout_ms elapses, and a + blocked lookup means a process serving nothing else in the + meantime. A LAN pull takes a couple of milliseconds and the + negative cache absorbs retransmits, but that is a statement about + the common case, not a guarantee under load. Enable it for + maintenance, migration or test paths; a startup warning repeats + this when it is on. + + + + Default value is 0 (disabled). + + +
+ + + + + +
+ +
+ DB persistence + + With set, a whole collection can be + saved to and loaded from an SQL backend. A save is a full snapshot: + the collection's rows are deleted and every live entry is re-inserted, + with its TTL stored as an absolute wall-clock time so it survives a + restart (already-expired entries are skipped on both save and load). + Native counters round-trip as their decimal value. + + + Trigger it on demand with the perf_save / + perf_load MI commands, or automatically via + . This is single-node durability; + for cross-node sharing over the same DB, see + (still not per-operation replication). + + + + A save/load is a full, blocking snapshot of the + collection: one SQL statement per entry, run synchronously in the + process that issued it. On a large collection (this module is built + for millions of entries) or a slow backend such as + db_text or db_sqlite, that + can take a long time and stall that process for its duration. Treat + it as a maintenance / bootstrap operation - + startup warm-up, shutdown flush, an occasional snapshot or a + perf_sync refresh - never on a per-request path + and not on a tight timer. Frequent whole-collection persistence is an + anti-pattern; if you need durable per-key writes on every operation, + this is the wrong tool. + + + The table (default cachedb_perf) needs these columns: + +collection string - the collection name +pkey string - the cache key +pvalue binary - the value (BLOB; binary-safe) +expires int - absolute unix expiry, 0 = never + +
+ +
+ Cluster sync + + Built on the same DB: with + set, perf_sync (MI command and script function) + saves a collection to the DB and then signals every node in the + cluster to reload it from there. The signal is one small message per + sync - not per cache operation - so it costs + nothing on the hot path. + + + + A reload overwrites a peer's copy from the DB, so + perf_sync is for single-writer / read-replica + topologies: one node (or the application writing the DB directly) is + the authority for a collection, the others refresh from it. A node + that also takes its own local writes would lose the unsaved ones on a + reload - this is convergence to a shared source of truth, not a merge + of divergent copies, and deliberately not per-operation replication. + + + + A node that reloads because of a peer's perf_sync + raises E_CACHEDB_PERF_SYNCED. With no clusterer + or sync_cluster_id 0, perf_sync + still saves to the DB, just without the peer signal. + + + When active, the module's capability shows in the clusterer's + clusterer_list_cap MI command as + cachedb-perf-sync. Note that the + state the clusterer reports there + (Ok) only means the capability is registered and + enabled: this module does not take part in the clusterer's + startup data-sync, so that field never reflects whether the caches + have converged. It would read not synced only if the + capability were disabled administratively. To see the sync activity + itself, use the last_sync_out / + last_sync_in / last_sync_source + fields of perf_stats, which report how many + seconds ago this node last pushed a snapshot and last reloaded one at + a peer's request (-1 = never), or subscribe to + E_CACHEDB_PERF_SYNCED. Between syncs the nodes + are expected to differ - convergence is on demand, by design. + + + Because it runs a full save first, perf_sync + carries the same blocking cost as perf_save (see + the warning under ): it is an occasional + refresh, not something to fire on a timer or per request. + + + Cluster-sync setup (one authority, two read replicas) + +# on every node - clusterer must load before cachedb_perf (the module +# declares a soft dependency, so init order is handled either way): +loadmodule "clusterer.so" +modparam("clusterer", "my_node_id", 1) # 2 and 3 on the other nodes +... +loadmodule "cachedb_perf.so" +modparam("cachedb_perf", "cache_collections", "profiles") +modparam("cachedb_perf", "db_url", "mysql://opensips:pw@dbhost/opensips") +modparam("cachedb_perf", "sync_cluster_id", 1) + +# on the authority node, after it has updated the "profiles" collection: +# opensips-cli -x mi cachedb_perf:perf_sync profiles +# -> saves "profiles" to the DB, signals nodes 2 and 3 to reload it +# +# or from script (e.g. after a reload route), same save-then-broadcast: +# perf_sync("profiles"); +# +# the replicas raise E_CACHEDB_PERF_SYNCED when they finish reloading: +event_route[E_CACHEDB_PERF_SYNCED] { + xlog("L_INFO", "reloaded $param(collection) from node $param(source_node)\n"); +} + + +
+ +
+ Degraded operation: what each surface does without its modules + + The cache itself never depends on the cluster plane. Whatever is + missing, the module starts, serves node-local traffic, and says at + startup exactly what it turned off. There are two degraded modes worth + knowing precisely; every output below is captured from a live instance, + not paraphrased. + + + Mode 1 - clusterer loaded, clusterer_controller absent. + The only thing lost is the clctr transport; pull and sync are fully + functional over the clusterer's BIN links. One warning at startup, + worded for the reason the controller is missing: + + +# controller not in this build: +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but this + build carries no clusterer_controller support - falling + back to 'bin' +# controller in the build but not loaded: +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but + clusterer_controller is not loaded - falling back to 'bin' + + + Mode 2 - neither module available. Pull and sync + are disabled entirely; the cache runs purely node-local. Three + warnings at startup: + + +WARNING:cachedb_perf:mod_init: clusterer module not available - + the cluster features are disabled; load clusterer before + cachedb_perf +WARNING:cachedb_perf:mod_init: pull_transport 'clctr' but this + build carries no clusterer_controller support - falling + back to 'bin' +WARNING:cachedb_perf:mod_init: replicate_collections is set but + the cluster is not available (needs sync_cluster_id + + clusterer) - cross-node pull disabled + + + Unaffected in both modes: every local surface. + The cachedb script API (cache_store / cache_fetch / cache_add / + cache_sub / cache_remove on "perf"), the glob functions (perf_del, + perf_mget, perf_mget_json), the local MI set (perf_get / perf_set / + perf_probe / perf_keys / perf_scan / perf_dump / perf_ttl / perf_del / + perf_stats / perf_stats_reset), DB persistence (perf_save / perf_load, + db_mode), and the four local events (E_CACHEDB_PERF_EXPIRED / NOMEM / + GROWN / MEM_DEGRADED). + + + perf_pull - in mode 1 the pull runs over bin; on a + single node (or when no peer holds the key) it reports where the answer + did not come from. In mode 2 it refuses: + + +# mode 1: +{ "source": "no-answer" } + +# mode 2: +{ "code": 500, + "message": "cross-node pull not active + (replicate_collections)" } + + + perf_cluster_probe - mode 1 probes over bin (on a + lone node: error 500, "could not start the probe - no peers, or no free + pull slot"); mode 2 answers error 400, "cross-node pull is not active + for this collection (replicate_collections)". + + + perf_sync (MI and script function) - never fails + for cluster reasons. Mode 1 saves and signals the peers; mode 2 + degrades to the DB save alone and says so: + + +# mode 1: +{ "collections": 2, "saved": 1, "broadcast": 2 } + +# mode 2: +{ "collections": 2, "saved": 1, "broadcast": 0, + "note": "cluster sync inactive (no clusterer / + cluster_id 0) - saved to the DB only" } + + + perf_stats - the per-collection and memory + sections are identical in both modes (including the + pulled_from_cluster / served_to_cluster counters, which simply stay 0). + The difference is the cluster object: present in + mode 1 (ids, membership, the pull counters and slot count, and a + topology array), absent in mode 2. + + +# mode 1 only: +"cluster": { + "cluster_id": 1, "my_node_id": 1, "peers_up": 0, + ... + "pull_slots": 64, + "topology": [ { "node_id": 1, "role": "self", "membership": "up" } ] +} + + + cache_fetch with pull_on_miss=1 - mode 1: a miss + on an opted-in collection blocks up to pull_timeout_ms asking the + cluster, exactly as documented under . + Mode 2: no pull exists, so a miss is a plain immediate miss - no + blocking, no timeout, no negative cache. + + + E_CACHEDB_PERF_SYNCED - only ever raised when a + peer's sync signal arrives, so it keeps firing in mode 1 and can never + fire in mode 2. Subscribing to it costs nothing either way. + +
+ +
+ Exported Statistics + + All counters are aggregated per-process and summed only when read, + so instrumentation never touches a shared cache line on the hot + path. Query with get_statistics cachedb_perf:. + + + hits / misses - fetch outcomes (expired counts as a miss) + stores / removes - write and explicit delete operations (removes counts only remove/perf_del, never TTL expiry) + expired - records reclaimed by the TTL expiry sweep (this is where timed-out keys are accounted, separate from removes) + destroyed - total records whose cells were freed back to the arena, from any cause; it equals removes + expired. Overwriting an existing key does not count here (the cell is reused in place), so entries = created - destroyed. A large gap between stores and entries with a small destroyed means most churn is same-key overwrites rather than expiry or deletes. + entries - live records across all collections + seqlock_retries - optimistic-read retries (the contention signal) + lock_fallbacks - reads that fell back to the bucket lock + arena_bytes / arena_chunks - memory taken from shm + memory_tier_probe / memory_tier_active - the huge-page tier the host was probed for, and the one the module's own arena actually runs on (1 hugetlb .. 4 plain 4K). They differ when arena_hugepage_mb could not be satisfied. + hugepage_arena_active, hugepage_arena_total_bytes, hugepage_arena_used_bytes, hugepage_arena_free_bytes - the module's own hugepage arena, which is SEPARATE from the OpenSIPS shm arena. Records live in one or the other, never both, so these do not add to arena_bytes. + + + Cross-node pull statistics. + These exist only when pull_on_miss is enabled. + They are grouped so that two identities hold, which is the point of + having them: every miss is accounted on the way out, and every + request is accounted on the way back. A missing counter here is not + cosmetic - it is a miss that vanished. + + + pulls_requested - misses that turned into a request on the wire. pulls_served - requests from peers this node answered. + pulls_suppressed - asks absorbed by an already-cached negative (pull_negative_ms); the second and later asker for a key a peer has already denied. + pulls_skip_notreplicated, pulls_skip_toolong, pulls_skip_nopeers, pulls_skip_noslot - misses refused at the gate before anything was sent. They are kept apart because each calls for a different action: pull is off for that collection; the key or collection name is too long to ask for at all; no live cluster member to ask; or the slot table is full and nothing was evictable. + pulls_received - a value came back. pulls_negative - a peer answered that it does not have the key. pulls_oversize - a peer has it but it exceeds the cluster transport limit. pulls_timed_out - nothing came back in pull_timeout_ms. pulls_send_failed - the transport refused the datagram, so no peer was ever asked. + pulls_stored - answers written into the cache. pulls_in_flight - requests outstanding right now (a gauge, not a total). + pulls_orphaned - a waiter gave up but the slot was kept in case the answer still arrives, for pull_linger_ms. Its outcomes: pulls_late_stored (arrived and was stored - convergence that would otherwise be lost), pulls_late_superseded (a local write had already filled the key), pulls_late_expired (arrived past the linger and was refused as stale), pulls_orphan_expired (no late answer ever came - the ordinary end of a timeout), pulls_orphan_evicted (the slot was reclaimed early because the pool ran dry). + pulls_abandoned - slots the reaper released because the caller never collected them. Distinct from a timeout, which the caller DID collect: a non-zero value here is a defect signal, not tuning. + pulls_foreign_cluster - pull messages that arrived on a controller cluster this module does not sync on and were refused. Non-zero means either a genuine multi-cluster node behaving correctly, or sync_cluster_id naming a cluster the controller does not manage; the accompanying warning tells the two apart. + + + The two identities: + + +misses = pulls_requested + pulls_suppressed + + pulls_skip_notreplicated + pulls_skip_toolong + + pulls_skip_nopeers + pulls_skip_noslot + +pulls_requested = pulls_received + pulls_negative + pulls_oversize + + pulls_timed_out + pulls_send_failed + pulls_in_flight + + + The second holds exactly on a two-node cluster. + pulls_negative is counted per REPLY, so with + more peers one request can raise it more than once. + + + The perf_stats MI command gives the same + figures broken down per collection, plus load factor, overflow + occupancy, retries-per-1k-reads, the memory-backing description and + hit_rate_pct - hits / (hits + misses) as a percentage. On a + healthy server the large majority of lookups hit (upwards of 80% + under steady dialog traffic); a persistently low or falling hit + rate means the cached state is being lost or is expiring before it + is used. The same guidance rides inline in the hit_rate_note field. + +
+ +
+ Exported MI Functions +
+ <function moreinfo="none">perf_stats</function> + + Reports per-collection statistics (entries, buckets, overflow, + hits/misses/stores/removes, load factor, seqlock retries and + retries-per-1k-reads) plus the arena occupancy and the achieved + memory tier. With no parameter it reports every collection; an + optional collection name restricts it to one. + + + Name: perf_stats. Parameters: + collection (optional). + + + <function>perf_stats</function> usage + +opensips-cli -x mi cachedb_perf:perf_stats +opensips-cli -x mi cachedb_perf:perf_stats th + + +
+ +
+ <function moreinfo="none">perf_stats_reset</function> + + The counters behind perf_stats - hits, + misses, stores, removes, expired, destroyed, retries - are + running totals since startup, so every rate derived from them is + a lifetime average. A burst of misses right after a restart, when + sequential requests arrive for dialogs older than the cache, + keeps dragging the hit rate down long after the cache has + recovered. This command re-baselines them so the next reading + covers a fresh interval, without restarting &osips;. + + + The counters themselves are not rewound: each process owns its + own counter cache line and must never have it written from + another process. Only a baseline is recorded, and the reported + figures are the difference. Live gauges - entries, buckets, + overflow, load factor and the arena figures - are read from + current state rather than from the counters, so a reset does not + disturb them. + + + With no parameter every collection is reset; an optional + collection name restricts it to one. + + + Name: perf_stats_reset. Parameters: + collection (optional). + + + <function>perf_stats_reset</function> usage + +opensips-cli -x mi cachedb_perf:perf_stats_reset +opensips-cli -x mi cachedb_perf:perf_stats_reset th + + +
+ +
+ Key introspection: perf_keys / perf_scan / perf_dump / perf_get / perf_probe / perf_pull / perf_set / perf_ttl / perf_del + + These give an operator the visibility that + cachedb_local lacks. All are lock-free: the + walkers take no bucket locks (seqlock reads), so unlike a + cachedb_local key scan they never stall SIP + traffic. Every command carries the perf_ prefix, + matching the script functions and staying clear of the core's bare + get/set. The optional + collection selects the table; omitted, it is the + groupless cachedb_url's collection. + + + perf_keys <glob> [collection] [limit] + - names (and TTL) of the keys matching a shell glob, bounded (default + 1000; the reply carries a note when it truncates). The + KEYS equivalent. + perf_scan <cursor> [glob] [count] + - cursored incremental iteration with Redis SCAN + semantics over the default collection: start with cursor 0 and + repeat with the returned cursor until it comes back 0. An entry + present throughout is returned at least once; count + bounds the buckets visited per call. This is the answer for a large + cache, where perf_keys would truncate. + perf_dump <glob> [collection] [limit] + - like perf_keys but includes the values; + values are opt-in, never the default. + perf_get <key> [collection] + - one key: its value, remaining TTL (-1 = never) and size. + perf_pull <key> [collection] + - fetch one key from the cluster, for a collection listed in + replicate_collections. Reports where the answer + came from: local (this node had it after all), + cluster (a peer had it, with its remaining + TTL), absent (every peer answered, none has + it) or no-answer (nobody answered in time, or + there was nobody to ask). The last two are deliberately different: + absence is a fact only when the whole cluster has said so. + + perf_probe <key> [collection] + - is the key here: its size and remaining TTL, but never the value. + Not merely a cheaper perf_get - it shares the + whole read path (same optimistic loop, lock fallback and expiry + rules) and stops before the copy-out, so it cannot disagree with a + read about whether a key is present; it allocates nothing and never + touches the record's payload. Use it to answer do you have + this key?, where a read would pay for bytes nobody + wants. + perf_set <key> <value> [ttl] [collection] + - write one key; ttl is seconds (0 or omitted = + never expires). + perf_ttl <glob> <ttl> [collection] + - re-arm the TTL of every key matching the glob without rewriting its + value (one atomic expiry store under the bucket lock, so lock-free + readers are undisturbed); ttl is seconds (0 = + never). Returns the count updated; a literal key matches exactly + one. + perf_del <glob> [collection] + - delete every key matching the glob; returns the count. The MI face + of the perf_del() script function. + + + introspection usage + +opensips-cli -x mi cachedb_perf:perf_keys "session-*" +opensips-cli -x mi cachedb_perf:perf_keys "session-*" th 50 +opensips-cli -x mi cachedb_perf:perf_scan 0 +opensips-cli -x mi cachedb_perf:perf_scan 384 "user-*" 128 +opensips-cli -x mi cachedb_perf:perf_dump "profile-*" +opensips-cli -x mi cachedb_perf:perf_get session-abc123 +opensips-cli -x mi cachedb_perf:perf_set greeting hello 300 +opensips-cli -x mi cachedb_perf:perf_ttl "session-*" 1800 +opensips-cli -x mi cachedb_perf:perf_del "session-abc*" + + +
+ +
+ <function moreinfo="none">perf_save / perf_load</function> + + Persist a collection to, or restore it from, the + backend (see + ). With no argument they operate on + every declared collection; with a collection name, only that one. + The reply reports how many collections and entries were written or + read. + + + <function>perf_save</function> / <function>perf_load</function> usage + +opensips-cli -x mi cachedb_perf:perf_save +opensips-cli -x mi cachedb_perf:perf_save sessions +opensips-cli -x mi cachedb_perf:perf_load sessions + + +
+ +
+ <function moreinfo="none">perf_sync</function> + + Save a collection to the DB and signal the cluster to reload it (see + ); all declared collections if none is + named. Also available as a script function, + perf_sync([collection]). + + + <function>perf_sync</function> usage + +opensips-cli -x mi cachedb_perf:perf_sync sessions + + +
+
+ +
+ Exported Events + + Every event is gated by evi_probe_event(), so + with no subscriber it costs a single shared read and nothing more; + none of them sit on the lock-free get/set path. + +
+ <function moreinfo="none">E_CACHEDB_PERF_EXPIRED</function> + + Raised by the sweep for each expired record it reclaims, but only for + the collections named in + (opt-in, since a + high-churn collection reaps in bulk and delivery is synchronous). + Parameters: collection, key. + +
+
+ <function moreinfo="none">E_CACHEDB_PERF_NOMEM</function> + + Raised when a write is dropped because the arena is full - the cache + is out of memory and rejecting stores. One event per dropped write + (subscribers should expect bursts under memory pressure). Parameters: + collection, key, + size (the value's byte length). + +
+
+ <function moreinfo="none">E_CACHEDB_PERF_GROWN</function> + + Raised by the maintenance timer after it grows a collection's table. + Parameters: collection, + prev_buckets, buckets, + splits, entries. + +
+
+ <function moreinfo="none">E_CACHEDB_PERF_MEM_DEGRADED</function> + + Raised once at startup when + arena_hugepage_mb was set but the arena settled + on a tier below hugetlb (missing vm.nr_overcommit_hugepages, + for instance) - the node is running slower than intended. Parameters: + requested_mb, tier (1 hugetlb + .. 4 plain 4K), backing (its description), + overcommit_pages. + +
+
+ <function moreinfo="none">E_CACHEDB_PERF_SYNCED</function> + + Raised on a node that reloaded a collection from the DB because a peer + issued perf_sync (see ). + Parameters: collection, + source_node (the cluster id of the node that + issued the sync). + +
+
+ +
+ Exported Functions + + Single-key operations go through the core cache functions + (cache_store(), + cache_fetch(), ...) with the + perf backend. The functions below are the module's + own glob (multi-key) operations. All of them match keys with + shell-style globs (fnmatch), walk the table + lock-free and give the Redis SCAN class of guarantee: an entry + mutated concurrently may be seen once, twice or not at all. + Unlike cachedb_local's + cache_remove_chunk(), these are + perf_-prefixed - scripts migrating from + cachedb_local must rename those calls. + When the optional collection argument is + omitted, they operate on the collection of the default + (groupless) cachedb_url - exactly where + cache_store("perf", ...) writes. + + +
+ <function moreinfo="none">perf_del(glob[, collection])</function> + + Deletes every key matching the glob (expired entries included). + Returns the number of keys removed, or -1 (false) if none + matched. + + + <function>perf_del()</function> usage + +... +perf_del("session-*"); +perf_del("th-*", "th"); +... + + +
+ +
+ <function moreinfo="none">perf_mget(glob, keys_avp, vals_avp[, collection[, limit]])</function> + + Returns every live key/value pair matching the glob into two + writable variables (use AVPs - each match adds one value to + each, and the indexes correspond pairwise; ordering is + unspecified). limit bounds the number of + matches, default 1000, 0 = unbounded. Returns the match count, + or -1 (false) if none matched. + + + <function>perf_mget()</function> usage + +... +if (perf_mget("user-*", $avp(k), $avp(v))) { + xlog("first match: $(avp(k)[0]) = $(avp(v)[0])\n"); +} +... + + +
+ +
+ <function moreinfo="none">perf_mget_json(glob, dst_var[, collection[, limit]])</function> + + Like perf_mget(), but returns all matches + as one JSON object {"key":"value",...} in a + single writable variable ({} when nothing + matches). Quote, backslash and control bytes are escaped, so + binary values survive; bytes above 0x7F pass through unescaped - + strict JSON consumers therefore need UTF-8 values. Returns the + match count, or -1 (false) if none matched. + + + <function>perf_mget_json()</function> usage + +... +if (perf_mget_json("user-*", $var(blob), , 100)) + xlog("users: $var(blob)\n"); +... + + +
+ +
+ + diff --git a/modules/cachedb_perf/pcache_arena.c b/modules/cachedb_perf/pcache_arena.c new file mode 100644 index 00000000000..d6acdfa0cbc --- /dev/null +++ b/modules/cachedb_perf/pcache_arena.c @@ -0,0 +1,652 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include +#include + +#include "../../dprint.h" +#include "../../locking.h" +#include "../../mem/mem.h" +#include "../../mem/shm_mem.h" + +#include "pcache_arena.h" +#include "pcache_mem.h" + +/* CP-20: MB to reserve for the huge-page arena; 0 = disabled (shm_malloc). + * Set by the cachedb_perf "arena_hugepage_mb" modparam. */ +int pcache_arena_hugepage_mb = 0; + +/* ~x1.5 ladder, all multiples of 32 so cells stay 8-aligned */ +static const unsigned int cell_sizes[PCACHE_NCLASSES] = { + 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, + 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536 +}; + +#define PCACHE_CHUNK_HDR 64 +#define PCACHE_CHUNK_SMALL (256 * 1024) /* cells <= 8K share 256K chunks */ +#define PCACHE_REFILL_BATCH 32 /* cells pulled from the global pool */ +#define PCACHE_PRIVATE_MAX 256 /* private stack size that triggers */ +#define PCACHE_DONATE 128 /* donation of this many cells */ + +typedef struct pcache_chunk { + struct pcache_chunk *next; /* global registry, append-only */ + unsigned int cls; /* immutable */ + unsigned int cell_size; + unsigned int cells; + /* padded to PCACHE_CHUNK_HDR; cells follow */ +} pcache_chunk_t; + +typedef struct pcache_region { + struct pcache_region *next; + unsigned long size; +} pcache_region_t; + +typedef struct pcache_arena { + gen_lock_t lock; /* slow paths only */ + pcache_chunk_t *chunks; + unsigned int nchunks; + unsigned long bytes; + pcache_region_t *regions; /* raw index regions */ + void *gpool[PCACHE_NCLASSES]; /* global free cells */ + unsigned int gpool_n[PCACHE_NCLASSES]; + unsigned long lo, hi; /* extent watermarks */ + + /* CP-20 huge-page reservation: a pre-fork, never-unmapped, 2M-aligned + * MAP_SHARED region. Chunks bump from it lock-free (atomic hoff); + * shm_malloc is the fallback once it is exhausted or if reserve fails */ + char *hbase; + unsigned long hsize; + volatile unsigned long hoff; + enum pcache_mem_tier htier; + unsigned long hlocked_mb; +} pcache_arena_t; + +/* per-process allocation state - pkg, lazily created, reset on fork */ +struct pcache_palloc { + struct { + char *bump; /* next unused cell in own chunk */ + unsigned int left; + void *free_head; /* private free stack */ + unsigned int nfree; + } cls[PCACHE_NCLASSES]; +}; + +static pcache_arena_t *arena; /* shm, set pre-fork */ +static struct pcache_palloc *my_palloc; /* pkg, per process */ +static unsigned char size2class[2049]; /* idx = ceil(size/32) */ + +/* free-list link: bytes 8..15, never byte 0 (the class id) */ +static inline void *cell_next(void *cell) +{ + return *(void **)((char *)cell + 8); +} + +static inline void cell_set_next(void *cell, void *next) +{ + *(void **)((char *)cell + 8) = next; +} + +/* global pool ops - arena lock must be held */ +static inline void gpool_push(int c, void *cell) +{ + cell_set_next(cell, arena->gpool[c]); + arena->gpool[c] = cell; + arena->gpool_n[c]++; +} + +static inline void *gpool_pop(int c) +{ + void *cell = arena->gpool[c]; + + if (cell) { + arena->gpool[c] = cell_next(cell); + arena->gpool_n[c]--; + } + return cell; +} + +/* + * THE CP-20 SEAM: every byte of arena memory funnels through here. + * If a huge-page reservation exists, bump from it lock-free (the caller's + * lock state varies - carve_chunk holds the arena lock, pcache_region_alloc + * does not - so an atomic bump is the only safe choice here); otherwise, or + * once it is exhausted, fall back to shm_malloc. Memory is NEVER returned + * while the server runs. + */ +static void *pcache_chunk_backing(size_t size) +{ + if (arena->hbase) { + unsigned long asz = (size + 63) & ~63UL; /* keep 64-aligned */ + unsigned long off = __atomic_fetch_add(&arena->hoff, asz, + __ATOMIC_RELAXED); + if (off + asz <= arena->hsize) + return arena->hbase + off; + /* exhausted: undo would race other bumpers, so just leave hoff + * past the end (further huge allocs also fall through) and use + * shm - correctness holds, we only lose the tail slack */ + } + return shm_malloc(size); +} + +static inline unsigned int chunk_size_for(int c) +{ + return cell_sizes[c] <= 8192 ? PCACHE_CHUNK_SMALL : cell_sizes[c] * 32; +} + +/* carve a new chunk for class @c - arena lock must be held. + * The class byte of every cell is stamped HERE, before the chunk is + * reachable by anyone - immutable from birth, so a stale reader can + * always trust it (DESIGN 3.2 copy-out rule 1). */ +static int carve_chunk(int c, struct pcache_palloc *pl) +{ + pcache_chunk_t *ch; + unsigned int size = chunk_size_for(c), i; + char *cells; + + ch = pcache_chunk_backing(size); + if (!ch) { + LM_ERR("no more shm memory for a %u byte chunk (class %d)\n", + size, c); + return -1; + } + + ch->cls = c; + ch->cell_size = cell_sizes[c]; + ch->cells = (size - PCACHE_CHUNK_HDR) / cell_sizes[c]; + + cells = (char *)ch + PCACHE_CHUNK_HDR; + for (i = 0; i < ch->cells; i++) + cells[(unsigned long)i * cell_sizes[c]] = (unsigned char)c; + + ch->next = arena->chunks; + arena->chunks = ch; + arena->nchunks++; + arena->bytes += size; + + if ((unsigned long)ch < arena->lo) + arena->lo = (unsigned long)ch; + if ((unsigned long)ch + size > arena->hi) + arena->hi = (unsigned long)ch + size; + + /* the whole chunk belongs to the carving process */ + pl->cls[c].bump = cells; + pl->cls[c].left = ch->cells; + + LM_DBG("class %d: new %u byte chunk, %u cells of %u\n", + c, size, ch->cells, cell_sizes[c]); + return 0; +} + +static struct pcache_palloc *get_palloc(void) +{ + if (!my_palloc) { + my_palloc = pkg_malloc(sizeof *my_palloc); + if (!my_palloc) { + LM_ERR("no more pkg memory\n"); + return NULL; + } + memset(my_palloc, 0, sizeof *my_palloc); + } + return my_palloc; +} + +int pcache_arena_init(void) +{ + int idx, c; + + arena = shm_malloc(sizeof *arena); + if (!arena) { + LM_ERR("no more shm memory\n"); + return -1; + } + memset(arena, 0, sizeof *arena); + arena->lo = ~0UL; + + if (!lock_init(&arena->lock)) { + LM_ERR("failed to init the arena lock\n"); + shm_free(arena); + arena = NULL; + return -1; + } + + /* size -> class LUT, built pre-fork and inherited */ + for (idx = 0; idx <= 2048; idx++) { + for (c = 0; c < PCACHE_NCLASSES; c++) + if (cell_sizes[c] >= (unsigned int)idx * 32) + break; + size2class[idx] = (unsigned char)c; /* NCLASSES = impossible */ + } + + /* CP-20: reserve the huge-page arena, pre-fork, if requested */ + if (pcache_arena_hugepage_mb > 0) { + arena->hsize = (unsigned long)pcache_arena_hugepage_mb << 20; + arena->hbase = pcache_mem_reserve(arena->hsize, &arena->htier, + &arena->hlocked_mb); + if (!arena->hbase) { + LM_WARN("huge-page arena reservation of %d MB failed; " + "falling back to shm_malloc (4K)\n", + pcache_arena_hugepage_mb); + arena->hsize = 0; + } else { + arena->lo = (unsigned long)arena->hbase; + arena->hi = (unsigned long)arena->hbase + arena->hsize; + LM_NOTICE("huge-page arena: %d MB on %s, %lu MB pinned from swapping\n", + pcache_arena_hugepage_mb, + pcache_mem_tier_str(arena->htier), arena->hlocked_mb); + } + } + + LM_DBG("arena ready: %d classes, %u B to %u B cells\n", + PCACHE_NCLASSES, cell_sizes[0], cell_sizes[PCACHE_NCLASSES-1]); + return 0; +} + +void *pcache_region_alloc(size_t size) +{ + pcache_region_t *rg; + unsigned long need = size + sizeof(pcache_region_t) + 64; + char *aligned; + + rg = pcache_chunk_backing(need); + if (!rg) { + LM_ERR("no more shm memory for a %lu byte region\n", need); + return NULL; + } + rg->size = need; + aligned = (char *)(((unsigned long)rg + sizeof(pcache_region_t) + 63) + & ~63UL); + + lock_get(&arena->lock); + rg->next = arena->regions; + arena->regions = rg; + arena->bytes += need; + if ((unsigned long)rg < arena->lo) + arena->lo = (unsigned long)rg; + if ((unsigned long)rg + need > arena->hi) + arena->hi = (unsigned long)rg + need; + lock_release(&arena->lock); + + return aligned; +} + +void pcache_arena_destroy(void) +{ + pcache_chunk_t *ch, *next; + pcache_region_t *rg, *rnext; + + if (!arena) + return; + + /* blocks carved from the huge reservation are part of one mmap - they + * must be munmap'd as a whole (below), never shm_free'd individually */ +#define IN_HARENA(_p) (arena->hbase && (char *)(_p) >= arena->hbase && \ + (char *)(_p) < arena->hbase + arena->hsize) + + for (rg = arena->regions; rg; rg = rnext) { + rnext = rg->next; + if (!IN_HARENA(rg)) + shm_free(rg); + } + for (ch = arena->chunks; ch; ch = next) { + next = ch->next; + if (!IN_HARENA(ch)) + shm_free(ch); + } + if (arena->hbase) + munmap(arena->hbase, arena->hsize); +#undef IN_HARENA + lock_destroy(&arena->lock); + shm_free(arena); + arena = NULL; + + if (my_palloc) { + pkg_free(my_palloc); + my_palloc = NULL; + } +} + +void pcache_arena_child_init(void) +{ + struct pcache_palloc *pl = my_palloc; + + if (!pl) + return; + + /* + * After fork every child holds a COW copy of the parent's private + * allocator state - the SAME bump pointer and the SAME free-list cell + * addresses. A child must not keep them (two processes bumping one + * chunk would hand out the same cell), and it must NOT donate them to + * the global pool either: every child inherited the identical copy, so + * each would push the same physical cells, landing one cell on the free + * list N times - later popped by several processes at once and written + * through concurrently (the CP-16 corruption: a value byte overwrites a + * neighbour's class id, and the next free reads an impossible class). + * + * The leftover cells belong to the parent. The child simply discards + * its inherited copy and starts empty, carving its own chunk on first + * use. The parent keeps its own small hoard. + * + * Bug fixed here (2026-08-07): this function's OWN comment already + * said "discards", but the code called pkg_free(pl) anyway - freeing + * pl (the pcache_palloc struct itself) is exactly the same class of + * mistake the comment warns about for its internal free-list cells: + * pl is COW-shared with the parent and every sibling child inherited + * the identical pointer, so pkg_free() is a WRITE into that shared + * page (hg_cell_free()/cell_set_next() links it into a free list). + * Under HG_MALLOC's hugepage-backed pkg arena this write-triggered + * COW fault reproducibly SIGBUSed (mem/hg_arena.c:98, always via + * cachedb_perf.c child_init -> here), first surfaced when a TCP-based + * protocol (proto_bin, for clusterer_controller) made this fork/free + * path run under HG_MALLOC for the first time. Fix: just drop the + * reference, exactly as documented - no free, no donation, nothing. + * pl's memory is reclaimed for free when the child process exits. + */ + my_palloc = NULL; +} + +void *pcache_cell_alloc(unsigned int size) +{ + struct pcache_palloc *pl; + void *cell; + unsigned int got; + int c; + + if (size > PCACHE_CELL_MAX) { + LM_DBG("%u bytes exceeds the largest cell (%d)\n", + size, PCACHE_CELL_MAX); + return NULL; + } + c = size2class[(size + 31) >> 5]; + + pl = get_palloc(); + if (!pl) + return NULL; + + /* fast paths: no locks, no shared lines */ + cell = pl->cls[c].free_head; + if (cell) { + pl->cls[c].free_head = cell_next(cell); + pl->cls[c].nfree--; + return cell; + } + if (pl->cls[c].left) { + cell = pl->cls[c].bump; + pl->cls[c].bump += cell_sizes[c]; + pl->cls[c].left--; + return cell; + } + + /* slow path: refill from the global pool, else carve a chunk */ + lock_get(&arena->lock); + for (got = 0; got < PCACHE_REFILL_BATCH; got++) { + cell = gpool_pop(c); + if (!cell) + break; + cell_set_next(cell, pl->cls[c].free_head); + pl->cls[c].free_head = cell; + pl->cls[c].nfree++; + } + if (!got && carve_chunk(c, pl) < 0) { + lock_release(&arena->lock); + return NULL; + } + lock_release(&arena->lock); + + cell = pl->cls[c].free_head; + if (cell) { + pl->cls[c].free_head = cell_next(cell); + pl->cls[c].nfree--; + return cell; + } + cell = pl->cls[c].bump; + pl->cls[c].bump += cell_sizes[c]; + pl->cls[c].left--; + return cell; +} + +void pcache_cell_free(void *cell) +{ + struct pcache_palloc *pl; + unsigned int c = *(unsigned char *)cell, i; + void *d; + + if (c >= PCACHE_NCLASSES) { + /* the class byte was clobbered - freeing through it would + * corrupt the pools; leak the cell and shout instead */ + LM_CRIT("cell %p carries invalid class %u - leaking it\n", + cell, c); + return; + } + + pl = get_palloc(); + if (!pl) { + /* cannot even track it privately - hand it to the pool */ + pcache_cell_free_global(cell); + return; + } + + cell_set_next(cell, pl->cls[c].free_head); + pl->cls[c].free_head = cell; + pl->cls[c].nfree++; + + /* keep hoarding bounded: donate half once over the threshold */ + if (pl->cls[c].nfree > PCACHE_PRIVATE_MAX) { + lock_get(&arena->lock); + for (i = 0; i < PCACHE_DONATE; i++) { + d = pl->cls[c].free_head; + pl->cls[c].free_head = cell_next(d); + pl->cls[c].nfree--; + gpool_push(c, d); + } + lock_release(&arena->lock); + } +} + +void pcache_cell_free_global(void *cell) +{ + unsigned int c = *(unsigned char *)cell; + + if (c >= PCACHE_NCLASSES) { + LM_CRIT("cell %p carries invalid class %u - leaking it\n", + cell, c); + return; + } + lock_get(&arena->lock); + gpool_push(c, cell); + lock_release(&arena->lock); +} + +unsigned int pcache_cell_bound(const void *cell) +{ + unsigned char c = *(const unsigned char *)cell; + + if (c >= PCACHE_NCLASSES) + return 0; + return cell_sizes[c]; +} + +void pcache_arena_extents(unsigned long *lo, unsigned long *hi) +{ + /* unlocked on purpose: both only ever grow outward. A reader mixing + * a fresh lo with a stale hi sees a subset - the check then fails + * closed (treated as invalid) for a moment during a chunk carve */ + *lo = arena->lo; + *hi = arena->hi; +} + +void pcache_arena_stats(unsigned int *nchunks, unsigned long *bytes) +{ + lock_get(&arena->lock); + *nchunks = arena->nchunks; + *bytes = arena->bytes; + lock_release(&arena->lock); +} + +/* the tier the huge-page reservation actually got (CP-11 MEM_DEGRADED) - + * distinct from pcache_mem.tier, which is the optimistic probe; with no + * reservation (arena_hugepage_mb=0 or a failed reserve) there is no arena + * to have a tier, reported as PCACHE_MEM_NO_ARENA */ +int pcache_arena_tier(void) +{ + /* No reservation is NOT the same as "backed by 4K pages": with no + * dedicated arena everything goes through the core's shm_malloc(), so + * the real backing is the CORE allocator's - 2M hugepages under + * HG_MALLOC. Returning PCACHE_MEM_4K here reported a property of an + * arena that does not exist, and was read live as "the cache is on + * small pages" while it was actually on hugepages. */ + return arena->hbase ? (int)arena->htier : PCACHE_MEM_NO_ARENA; +} + +/* + * Capacity of the DEDICATED arena_hugepage_mb reservation specifically - + * distinct from pcache_arena_stats()'s bytes/nchunks, which count total + * cachedb_perf usage regardless of backing (dedicated reservation OR the + * shm_malloc fallback, whichever actually served each allocation). + * + * @active is 0 whenever arena_hugepage_mb was never set (or the reserve + * failed) - callers MUST check it before trusting total/used/free, since + * 0/0/0 alone cannot distinguish "no dedicated reservation exists" from + * "a reservation exists and happens to be still empty". + */ +void pcache_arena_hugepage_capacity(int *active, unsigned long *total, + unsigned long *used, unsigned long *free) +{ + if (!arena->hbase) { + *active = 0; + *total = 0; + *used = 0; + *free = 0; + return; + } + + *active = 1; + *total = arena->hsize; + *used = arena->hoff; + *free = arena->hsize - arena->hoff; +} + + +/* + * startup selftest (modparam "arena_selftest"): exercises class mapping, + * the stamp/bound contract, LIFO reuse, chunk growth, donation, refill, + * extents and the oversize edge, on the pre-fork process. Ends by + * donating everything through pcache_arena_child_init(), which is the + * fork-reset path - so that gets exercised too. + */ +#define CHK(cond, ...) \ + do { \ + if (!(cond)) { \ + LM_ERR("arena selftest FAILED: " __VA_ARGS__); \ + return -1; \ + } \ + } while (0) + +int pcache_arena_selftest(void) +{ + void *a, *b, **ptrs; + unsigned long lo, hi, by0, by1; + unsigned int n0, n1, n2, i; + const unsigned int N = 5000; + int c; + + /* class mapping, stamp, bound, LIFO reuse, boundary crossing */ + for (c = 0; c < PCACHE_NCLASSES; c++) { + a = pcache_cell_alloc(cell_sizes[c]); + CHK(a != NULL, "alloc(%u) failed\n", cell_sizes[c]); + CHK(*(unsigned char *)a == c, "class stamp %d != %d\n", + *(unsigned char *)a, c); + CHK(pcache_cell_bound(a) == cell_sizes[c], + "bound %u != %u\n", pcache_cell_bound(a), cell_sizes[c]); + memset((char *)a + 1, 0xAB, cell_sizes[c] - 1); + pcache_cell_free(a); + b = pcache_cell_alloc(cell_sizes[c]); + CHK(b == a, "no LIFO reuse in class %d\n", c); + pcache_cell_free(b); + + if (c < PCACHE_NCLASSES - 1) { + a = pcache_cell_alloc(cell_sizes[c] + 1); + CHK(*(unsigned char *)a == c + 1, + "size %u not in class %d\n", cell_sizes[c] + 1, c + 1); + pcache_cell_free(a); + } + } + + /* oversize and zero */ + CHK(pcache_cell_alloc(PCACHE_CELL_MAX + 1) == NULL, "oversize passed\n"); + a = pcache_cell_alloc(0); + CHK(a && *(unsigned char *)a == 0, "zero-size alloc broken\n"); + pcache_cell_free(a); + + /* bulk: multiple chunks, uniqueness, extents */ + ptrs = pkg_malloc(N * sizeof *ptrs); + CHK(ptrs != NULL, "no pkg for the pointer array\n"); + pcache_arena_stats(&n0, &by0); + for (i = 0; i < N; i++) { + ptrs[i] = pcache_cell_alloc(64); + if (!ptrs[i]) { + pkg_free(ptrs); + CHK(0, "bulk alloc %u failed\n", i); + } + *(unsigned int *)((char *)ptrs[i] + 8) = i; + } + pcache_arena_stats(&n1, &by1); + CHK(n1 > n0, "no chunk growth over %u allocs\n", N); + pcache_arena_extents(&lo, &hi); + for (i = 0; i < N; i++) { + CHK(*(unsigned int *)((char *)ptrs[i] + 8) == i, + "cell %u overlapped\n", i); + CHK((unsigned long)ptrs[i] >= lo && + (unsigned long)ptrs[i] + 64 <= hi, + "cell %u outside the extents\n", i); + } + for (i = 0; i < N; i++) + pcache_cell_free(ptrs[i]); + + /* the private stack must have donated past the threshold */ + CHK(arena->gpool_n[0] > 0, "no donation after %u frees\n", N); + + /* full reuse: no new chunks on the second pass (refill path) */ + for (i = 0; i < N; i++) { + ptrs[i] = pcache_cell_alloc(64); + if (!ptrs[i]) { + pkg_free(ptrs); + CHK(0, "realloc %u failed\n", i); + } + } + pcache_arena_stats(&n2, &by1); + CHK(n2 == n1, "reuse pass grew chunks: %u -> %u\n", n1, n2); + for (i = 0; i < N; i++) + pcache_cell_free(ptrs[i]); + pkg_free(ptrs); + + /* fork-reset path: donate everything, then allocate fresh */ + pcache_arena_child_init(); + CHK(my_palloc == NULL, "child reset kept state\n"); + a = pcache_cell_alloc(64); + CHK(a != NULL, "alloc after child reset failed\n"); + pcache_cell_free(a); + + LM_NOTICE("arena selftest: PASS (%u chunks, %lu bytes, " + "%d classes)\n", n2, by1, PCACHE_NCLASSES); + return 0; +} diff --git a/modules/cachedb_perf/pcache_arena.h b/modules/cachedb_perf/pcache_arena.h new file mode 100644 index 00000000000..94c7bfe3950 --- /dev/null +++ b/modules/cachedb_perf/pcache_arena.h @@ -0,0 +1,95 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _PCACHE_ARENA_H_ +#define _PCACHE_ARENA_H_ + +/* + * Slab arena (DESIGN 3.3): entries live in fixed-size cells inside chunks + * taken from shm and NEVER returned while the server runs - that is what + * makes the lock-free read path (DESIGN 3.2) legal. A chunk is permanently + * bound to one size class; cells never straddle or move. + * + * The cell contract: + * - byte 0 of every cell is the CLASS ID, stamped for the whole chunk at + * carve time and never written again. Callers lay their record out + * with byte 0 as a read-only class field. This is how the copy-out + * clamp finds its bound through a stale pointer without aligned chunks: + * pcache_cell_bound() range-checks the byte and returns the cell size. + * - bytes 8..15 carry the free-list link while a cell is free; a live + * cell owns everything from byte 1 up. + * + * Allocation state is per-process (pkg, lazy): a bump chunk plus a private + * free stack per class - zero shm traffic and zero atomics on the fast + * path. Owner frees go to the private stack (LIFO reuse); oversized + * stacks donate half to a per-class global pool, which also serves refills + * and takes cross-process frees (expiry / maintenance worker). + */ + +#define PCACHE_CELL_MAX 65536 /* largest cell; bigger allocs fail (v1) */ +#define PCACHE_NCLASSES 21 + +int pcache_arena_init(void); +void pcache_arena_destroy(void); + +/* reset inherited allocator state after fork: donates any pre-fork bump + * chunk / private cells to the global pool. Two processes must never + * share a bump pointer. */ +void pcache_arena_child_init(void); + +/* a cell of at least @size bytes (including the class byte), or NULL if + * size > PCACHE_CELL_MAX or shm is exhausted */ +void *pcache_cell_alloc(unsigned int size); + +/* a raw, 64-byte-aligned, never-freed region for index structures (bucket + * segments, directories). Same backing seam and never-returned guarantee + * as chunks; NOT carved into cells and NOT zeroed. */ +void *pcache_region_alloc(size_t size); + +/* owner free: private stack of the calling process */ +void pcache_cell_free(void *cell); + +/* cross-process free (expiry sweep, maintenance worker): global pool */ +void pcache_cell_free_global(void *cell); + +/* clamp bound for a possibly-stale cell pointer: the cell size of the + * class in byte 0, or 0 if the byte is not a valid class id */ +unsigned int pcache_cell_bound(const void *cell); + +/* monotone address watermarks over all chunks (DESIGN 3.2 rule 2) */ +void pcache_arena_extents(unsigned long *lo, unsigned long *hi); + +void pcache_arena_stats(unsigned int *nchunks, unsigned long *bytes); + +/* the memory tier the huge-page reservation actually achieved (1 hugetlb .. + * 4 plain 4K), as opposed to the pcache_mem.tier probe - CP-11 */ +int pcache_arena_tier(void); + +/* see the implementation comment in pcache_arena.c - @active must be + * checked before trusting total/used/free */ +void pcache_arena_hugepage_capacity(int *active, unsigned long *total, + unsigned long *used, unsigned long *free); + +/* modparam-triggered startup selftest; returns -1 on any mismatch */ +int pcache_arena_selftest(void); + +#endif /* _PCACHE_ARENA_H_ */ diff --git a/modules/cachedb_perf/pcache_db.c b/modules/cachedb_perf/pcache_db.c new file mode 100644 index 00000000000..a2e2e5863ba --- /dev/null +++ b/modules/cachedb_perf/pcache_db.c @@ -0,0 +1,362 @@ +/* + * cachedb_perf - DB persistence (see pcache_db.h). + */ +#include + +#include "../../dprint.h" +#include "../../timer.h" +#include +#include "../../db/db.h" +#include "../../db/db_cap.h" +#include "../../config.h" /* SHUTDOWN_TIMEOUT */ + +/* warn when a snapshot takes longer than this - well inside the 60 s + * shutdown watchdog, so the warning arrives before the cliff */ +#define PCACHE_DB_SLOW_SAVE_SECS 10.0 + +#include "pcache_db.h" +#include "pcache_htable.h" + +static db_func_t pcache_dbf; +static int pcache_db_bound; +static str pcache_db_url; +static str pcache_db_table; + +/* + * A snapshot is one statement per row, so on a backend that commits (and + * fsyncs) each one separately it is slow enough to matter: a 30k-entry + * collection took ~60 s on db_sqlite and did not finish inside the shutdown + * watchdog, against ~5 s on db_redis. Wrapping the whole snapshot in one + * transaction removes the per-row commit, and - more importantly - makes it + * atomic: a save begins by deleting the collection, so an interrupted save + * would otherwise leave a partially written table where a complete one used + * to be. Uncommitted work is rolled back when the connection closes, so a + * failure or a kill leaves the previous snapshot intact. + * + * Only used where the driver exposes raw_query (SQL backends). Backends + * without it - db_redis - do not need it: they have no per-row commit to + * amortise. + */ +static int pcache_db_raw(db_con_t *dbh, const char *what) +{ + str q; + + q.s = (char *)what; + q.len = strlen(what); + return pcache_dbf.raw_query(dbh, &q, NULL); +} + +static int pcache_db_txn_begin(db_con_t *dbh) +{ + if (!DB_CAPABILITY(pcache_dbf, DB_CAP_RAW_QUERY) || !pcache_dbf.raw_query) + return -1; + + /* No one spelling works everywhere: SQLite and PostgreSQL take + * "BEGIN TRANSACTION", MySQL takes "START TRANSACTION". Try both rather + * than sniff the driver. (The bare "BEGIN" both would accept is not an + * option: db_sqlite's raw_query only reaches its exec path for + * statements at least as long as "select", so a 5-character statement is + * mis-parsed as a SELECT.) */ + if (pcache_db_raw(dbh, "BEGIN TRANSACTION") == 0) + return 0; + if (pcache_db_raw(dbh, "START TRANSACTION") == 0) + return 0; + + LM_DBG("backend did not accept a transaction - saving without one\n"); + return -1; +} + +static int pcache_db_txn_commit(db_con_t *dbh) +{ + return pcache_db_raw(dbh, "COMMIT"); +} + +/* one row per cache entry: (collection, pkey, pvalue, expires) */ +static str col_collection = str_init("collection"); +static str col_pkey = str_init("pkey"); +static str col_pvalue = str_init("pvalue"); +static str col_expires = str_init("expires"); + +int pcache_db_enabled(void) +{ + return pcache_db_bound; +} + +/* read a column as a str regardless of how the backend typed it - a TEXT + * column comes back DB_STRING from some drivers (sqlite), DB_STR from + * others, and the value is DB_BLOB. Returns -1 if NULL/unsupported. */ +static int db_col_str(const db_val_t *v, str *out) +{ + if (VAL_NULL(v)) + return -1; + switch (VAL_TYPE(v)) { + case DB_STR: + *out = VAL_STR(v); + break; + case DB_BLOB: + *out = VAL_BLOB(v); + break; + case DB_STRING: + out->s = (char *)VAL_STRING(v); + out->len = out->s ? strlen(out->s) : 0; + break; + default: + return -1; + } + return 0; +} + +int pcache_db_init(const str *db_url, const str *db_table) +{ + if (db_bind_mod(db_url, &pcache_dbf) < 0) { + LM_ERR("cannot bind to a database module for <%.*s> - is the " + "matching db_* module loaded?\n", db_url->len, db_url->s); + return -1; + } + if (!DB_CAPABILITY(pcache_dbf, + DB_CAP_QUERY | DB_CAP_INSERT | DB_CAP_DELETE)) { + LM_ERR("the database backend lacks query/insert/delete support\n"); + return -1; + } + pcache_db_url = *db_url; + pcache_db_table = *db_table; + pcache_db_bound = 1; + LM_INFO("DB persistence bound to <%.*s>, table <%.*s>\n", + db_url->len, db_url->s, db_table->len, db_table->s); + return 0; +} + +/* --- save --- */ + +struct db_save_ctx { + db_con_t *dbh; + str *coll; + unsigned int now_ticks; + long now_wall; + int n, err; +}; + +static int db_save_cb(const str *key, const str *val, unsigned int exp, + void *p) +{ + struct db_save_ctx *sc = p; + static db_key_t cols[4] = + { &col_collection, &col_pkey, &col_pvalue, &col_expires }; + db_val_t vals[4]; + + if (exp && exp <= sc->now_ticks) + return 0; /* skip already-expired */ + + memset(vals, 0, sizeof vals); + VAL_TYPE(&vals[0]) = DB_STR; VAL_STR(&vals[0]) = *sc->coll; + VAL_TYPE(&vals[1]) = DB_STR; VAL_STR(&vals[1]) = *(str *)key; + VAL_TYPE(&vals[2]) = DB_BLOB; VAL_BLOB(&vals[2]) = *(str *)val; + VAL_TYPE(&vals[3]) = DB_INT; + /* monotonic ticks -> absolute wall clock, so the TTL survives a reboot */ + VAL_INT(&vals[3]) = exp ? + (int)(sc->now_wall + (long)(exp - sc->now_ticks)) : 0; + + if (pcache_dbf.insert(sc->dbh, cols, vals, 4) < 0) { + LM_ERR("insert failed for key <%.*s>\n", key->len, key->s); + sc->err = 1; + return -1; /* stop the walk */ + } + sc->n++; + return 0; +} + +int pcache_db_save(pcache_col_t *col) +{ + db_con_t *dbh; + db_key_t wk[1] = { &col_collection }; + db_val_t wv[1]; + struct db_save_ctx sc; + struct timeval t0, t1; + double secs; + int txn; + + if (!pcache_db_bound) { + LM_ERR("no DB backend configured (set db_url)\n"); + return -1; + } + dbh = pcache_dbf.init(&pcache_db_url); + if (!dbh) { + LM_ERR("cannot open the DB connection\n"); + return -1; + } + if (pcache_dbf.use_table(dbh, &pcache_db_table) < 0) { + LM_ERR("use_table <%.*s> failed\n", + pcache_db_table.len, pcache_db_table.s); + pcache_dbf.close(dbh); + return -1; + } + + gettimeofday(&t0, NULL); + + /* One transaction for the whole snapshot where the backend supports it: + * it removes the per-row commit AND makes the delete+insert atomic, so an + * interrupted save cannot leave a half-written table behind. */ + txn = pcache_db_txn_begin(dbh) == 0; + + /* a snapshot replaces the previous one: clear this collection's rows */ + memset(wv, 0, sizeof wv); + VAL_TYPE(&wv[0]) = DB_STR; + VAL_STR(&wv[0]) = col->col_name; + if (pcache_dbf.delete(dbh, wk, NULL, wv, 1) < 0) { + LM_ERR("failed to clear old rows for <%.*s>\n", + col->col_name.len, col->col_name.s); + /* closing without COMMIT rolls back - the old snapshot survives */ + pcache_dbf.close(dbh); + return -1; + } + + memset(&sc, 0, sizeof sc); + sc.dbh = dbh; + sc.coll = &col->col_name; + sc.now_ticks = get_ticks(); + sc.now_wall = (long)time(NULL); + pcache_ht_iter(col->htable, db_save_cb, &sc); + + if (sc.err) { + /* leave the transaction uncommitted: the previous snapshot stands */ + LM_ERR("collection <%.*s>: save failed after %d rows - the previous " + "snapshot is left in place\n", + col->col_name.len, col->col_name.s, sc.n); + pcache_dbf.close(dbh); + return -1; + } + if (txn && pcache_db_txn_commit(dbh) < 0) { + LM_ERR("collection <%.*s>: could not commit %d rows - the previous " + "snapshot is left in place\n", + col->col_name.len, col->col_name.s, sc.n); + pcache_dbf.close(dbh); + return -1; + } + pcache_dbf.close(dbh); + + gettimeofday(&t1, NULL); + secs = (t1.tv_sec - t0.tv_sec) + (t1.tv_usec - t0.tv_usec) / 1e6; + LM_INFO("collection <%.*s>: saved %d entries in %.2f s (%.0f rows/s)%s\n", + col->col_name.len, col->col_name.s, sc.n, secs, + secs > 0 ? sc.n / secs : 0.0, + txn ? "" : " [no transaction - backend has no raw_query]"); + + /* A shutdown save runs inside SHUTDOWN_TIMEOUT (60 s); overrun it and the + * core aborts the process mid-write. Warn well before that, because the + * symptom otherwise is a snapshot that silently stops part-way. */ + if (secs > PCACHE_DB_SLOW_SAVE_SECS) + LM_WARN("collection <%.*s>: the snapshot took %.1f s for %d entries%s. " + "A save on shutdown has to finish within %d s or the core aborts " + "the process; the snapshot itself is safe (it is rolled back, " + "leaving the previous one) but no new one is written. Persist " + "fewer entries, or move to a backend that can hold the snapshot " + "in one transaction.\n", + col->col_name.len, col->col_name.s, secs, sc.n, + txn ? "" : " - and this backend took no transaction, so every " + "row was committed separately", + SHUTDOWN_TIMEOUT); + + return sc.n; +} + +/* --- load --- */ + +int pcache_db_load(pcache_col_t *col) +{ + db_con_t *dbh; + db_key_t qcols[3] = { &col_pkey, &col_pvalue, &col_expires }; + db_key_t wk[1] = { &col_collection }; + db_val_t wv[1]; + db_res_t *res = NULL; + db_row_t *rows; + db_val_t *v; + str key, val; + unsigned int now_ticks; + long now_wall; + int i, expires, remaining, n = 0, stale = 0; + + if (!pcache_db_bound) { + LM_ERR("no DB backend configured (set db_url)\n"); + return -1; + } + dbh = pcache_dbf.init(&pcache_db_url); + if (!dbh) { + LM_ERR("cannot open the DB connection\n"); + return -1; + } + if (pcache_dbf.use_table(dbh, &pcache_db_table) < 0) { + LM_ERR("use_table <%.*s> failed\n", + pcache_db_table.len, pcache_db_table.s); + pcache_dbf.close(dbh); + return -1; + } + + memset(wv, 0, sizeof wv); + VAL_TYPE(&wv[0]) = DB_STR; + VAL_STR(&wv[0]) = col->col_name; + if (pcache_dbf.query(dbh, wk, NULL, wv, qcols, 1, 3, NULL, &res) < 0) { + LM_ERR("query for <%.*s> failed\n", + col->col_name.len, col->col_name.s); + pcache_dbf.close(dbh); + return -1; + } + + now_ticks = get_ticks(); + now_wall = (long)time(NULL); + rows = RES_ROWS(res); + for (i = 0; i < RES_ROW_N(res); i++) { + v = ROW_VALUES(rows + i); + if (db_col_str(&v[0], &key) < 0 || db_col_str(&v[1], &val) < 0) + continue; + expires = VAL_NULL(&v[2]) ? 0 : VAL_INT(&v[2]); + + if (expires == 0) { + remaining = 0; /* never expires */ + } else { + remaining = expires - (int)now_wall; + if (remaining <= 0) { + stale++; /* already expired in the DB */ + continue; + } + } + if (pcache_ht_store(col->htable, &key, &val, + remaining ? now_ticks + (unsigned int)remaining : 0) < 0) { + LM_ERR("store of <%.*s> failed during load\n", + key.len, key.s); + continue; + } + n++; + } + pcache_dbf.free_result(dbh, res); + + /* Rows whose wall-clock expiry has passed are dead weight: nothing will + * ever load them, and without a save to rewrite the snapshot (a crash, or + * db_mode=1 which never writes) they would sit there forever. Drop them + * in one ranged delete now that the result set is released. expires=0 + * means "never expires", so it must be excluded explicitly - it would + * otherwise match the <= comparison. */ + if (stale > 0) { + db_key_t dk[3] = { &col_collection, &col_expires, &col_expires }; + db_op_t dop[3] = { OP_EQ, OP_GT, OP_LEQ }; + db_val_t dv[3]; + + memset(dv, 0, sizeof dv); + VAL_TYPE(&dv[0]) = DB_STR; VAL_STR(&dv[0]) = col->col_name; + VAL_TYPE(&dv[1]) = DB_INT; VAL_INT(&dv[1]) = 0; + VAL_TYPE(&dv[2]) = DB_INT; VAL_INT(&dv[2]) = (int)now_wall; + + if (pcache_dbf.delete(dbh, dk, dop, dv, 3) < 0) + LM_WARN("collection <%.*s>: could not remove %d stale entries " + "- they are ignored, but will be retried on the next " + "load and cleared by the next save\n", + col->col_name.len, col->col_name.s, stale); + else + LM_INFO("collection <%.*s>: removed %d stale entries\n", + col->col_name.len, col->col_name.s, stale); + } + + pcache_dbf.close(dbh); + LM_INFO("collection <%.*s>: loaded %d entries\n", + col->col_name.len, col->col_name.s, n); + return n; +} diff --git a/modules/cachedb_perf/pcache_db.h b/modules/cachedb_perf/pcache_db.h new file mode 100644 index 00000000000..0f4a3f1689f --- /dev/null +++ b/modules/cachedb_perf/pcache_db.h @@ -0,0 +1,30 @@ +/* + * cachedb_perf - DB persistence of whole collections to a db_* backend. + * + * The DB is a shared source of truth; the in-memory cache is a view over it. + * A collection is saved as a full snapshot (delete its rows, insert all live + * entries) and loaded back the same way. TTLs are stored as absolute wall- + * clock time, so they survive a restart (the cache's own expiry is in + * monotonic ticks, which reset on reboot). + */ +#ifndef _PCACHE_DB_H_ +#define _PCACHE_DB_H_ + +#include "../../str.h" +#include "cachedb_perf.h" + +/* bind the db_* module at @db_url and remember the table. 0 ok, -1 error. */ +int pcache_db_init(const str *db_url, const str *db_table); + +/* is a DB backend configured? */ +int pcache_db_enabled(void); + +/* save a collection as a full snapshot (DELETE its rows, then INSERT every + * live entry with an absolute-unix expiry). Returns rows written, -1 error. */ +int pcache_db_save(pcache_col_t *col); + +/* load a collection from the DB into its table, skipping rows that have + * already expired. Returns rows loaded, -1 error. */ +int pcache_db_load(pcache_col_t *col); + +#endif /* _PCACHE_DB_H_ */ diff --git a/modules/cachedb_perf/pcache_htable.c b/modules/cachedb_perf/pcache_htable.c new file mode 100644 index 00000000000..74fae5113a6 --- /dev/null +++ b/modules/cachedb_perf/pcache_htable.c @@ -0,0 +1,1930 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/* + * The table core (DESIGN 3.1/3.2/3.4): 64-byte buckets, 1-byte tags, + * lock-free optimistic reads under a per-bucket seqlock, writers under the + * bucket lock. Rules implemented here and not to be broken: + * + * - readers copy out inside the optimistic section and trust nothing + * until the version re-check; every length is clamped and every + * pointer extent-checked BEFORE use (3.2 copy-out rules) + * - a byte-identical set() that only refreshes the TTL skips the version + * bumps and the memcpy - one atomic expires store under the lock + * (2.7); readers of the bucket are undisturbed + * - no allocation and no free while holding a bucket lock (3.5b): + * replacement records are built before lock_get, dead records are + * freed after lock_release + * - on a miss the routing word is re-read (3.4): a completed split may + * have moved the key; writers re-verify routing after lock_get + * - full buckets overflow into a small chained side table behind one + * lock, gated by ovf_count so the common case costs one cached load; + * a key lives in its bucket or in overflow, never both + */ + +#include + +#include "../../dprint.h" +#include "../../hash_func.h" +#include "../../locking.h" +#include "../../pt.h" +#include "../../timer.h" +#include "../../mem/mem.h" + +#include "pcache_arena.h" +#include "pcache_htable.h" + +/* The selftest deliberately drives two rejection paths (a non-numeric add + * and an oversize store). Both log at L_ERR by design, which in a PASSING + * selftest reads as a real fault. This flag downgrades exactly those two + * messages while the selftest provokes them. It is only ever set pre-fork, + * single-threaded, from pcache_htable_selftest(), and is never set in normal + * operation. (The core set_proc_log_level() cannot be used here: it writes + * pt[process_no], and the process table does not exist yet at mod_init.) */ +static int st_expect_reject; + +#define PCACHE_REJECT_LOG(...) \ + do { \ + if (st_expect_reject) \ + LM_DBG(__VA_ARGS__); \ + else \ + LM_ERR(__VA_ARGS__); \ + } while (0) + + +#if defined(__x86_64__) || defined(__i386__) +#define pcache_pause() __builtin_ia32_pause() +#else +#define pcache_pause() do {} while (0) +#endif + +struct povf { + /* byte 0 is the arena class id (pcache_cell_free reads it from every + * cell) - it must never be overwritten, so the link pointer cannot + * live at offset 0 the way it briefly did (its low byte clobbered + * the class, sending frees to an out-of-range class -> pool + * corruption -> crashes in the donate walk) */ + unsigned char cls_reserved; + struct povf *next; + pcache_rec_t *rec; + unsigned int hash; +}; + +static inline unsigned char tag_of(unsigned int h) +{ + unsigned char t = (unsigned char)(h >> 24); + + return t ? t : 1; /* never t|1 - that halves the tag alphabet */ +} + +static inline unsigned int route_idx(pcache_htable_t *ht, unsigned int h, + uint64_t *route_out) +{ + /* acquire pairs with the release-publish in pcache_ht_split: seeing a + * new routing word implies the partner bucket's slots are visible */ + uint64_t r = __atomic_load_n(&ht->route, __ATOMIC_ACQUIRE); + unsigned int level = (unsigned int)(r >> 32); + unsigned int split = (unsigned int)r; + unsigned int idx = h & ((1U << level) - 1); + + if (idx < split) + idx = h & ((1U << (level + 1)) - 1); + *route_out = r; + return idx; +} + +static inline pcache_bucket_t *bucket_at(pcache_htable_t *ht, unsigned int idx) +{ + return &ht->seg[idx >> PCACHE_SEG_BITS][idx & (PCACHE_SEG_SIZE - 1)]; +} + +static inline unsigned int *hint_at(pcache_htable_t *ht, unsigned int idx) +{ + return &ht->hint_seg[idx >> PCACHE_SEG_BITS][idx & (PCACHE_SEG_SIZE - 1)]; +} + +/* under the bucket lock; only a LOWER expiry writes (TTL bumps raise) */ +static inline void hint_update(pcache_htable_t *ht, unsigned int idx, + unsigned int exp) +{ + unsigned int *h = hint_at(ht, idx); + + if (exp && (!*h || exp < *h)) + *h = exp; +} + +/* CP-06: plain increments on the calling process's own cache line */ +#define HT_ST(_ht, _f) \ + do { \ + if ((unsigned int)process_no < (_ht)->pstats_n) \ + (_ht)->pstats[process_no]._f++; \ + } while (0) + +#define HT_ST_ADD(_ht, _f, _n) \ + do { \ + if ((unsigned int)process_no < (_ht)->pstats_n) \ + (_ht)->pstats[process_no]._f += (_n); \ + } while (0) + +/* one 8-byte load of tags[6]+meta; 0x80 at byte i = tags[i] matches. + * The SWAR borrow can produce a false positive after a true match byte - + * filtered by the key compare, never a false negative. */ +static inline uint64_t tag_matches(const pcache_bucket_t *b, + unsigned char tag) +{ + /* uint64_t, not unsigned long: this word IS the 8 tag bytes, and the + * masks below are 64-bit constants. On an ILP32 target unsigned long + * is 4 bytes, so the memcpy would overflow and every constant would be + * truncated - the filter would return garbage rather than fail loudly. */ + uint64_t w, x; + + memcpy(&w, b->tags, 8); + x = w ^ (0x0101010101010101ULL * tag); + return (x - 0x0101010101010101ULL) & ~x & 0x0000808080808080ULL; +} + +/* meta helpers - writers only, under the bucket lock */ +static inline unsigned int bkt_used(const pcache_bucket_t *b) +{ + return b->meta & 0xF; +} + +static inline void bkt_set_used(pcache_bucket_t *b, unsigned int used) +{ + b->meta = (b->meta & ~0xF) | used; +} + +static inline void bkt_set_owner(pcache_bucket_t *b) +{ + b->meta = (b->meta & 0xF) | + (unsigned short)(((process_no + 1) & 0xFFF) << 4); +} + +static inline void bkt_clear_owner(pcache_bucket_t *b) +{ + b->meta &= 0xF; +} + +/* per-process copy-out scratch (3.2 rule 3) */ +static char *pcache_scratch; + +static char *get_scratch(void) +{ + if (!pcache_scratch) + pcache_scratch = pkg_malloc(PCACHE_CELL_MAX); + if (!pcache_scratch) + LM_ERR("no more pkg memory for the copy-out scratch\n"); + return pcache_scratch; +} + +/* + * Scan @b for @key under presumed-stable state: bounded reads only, so it + * is safe both inside an optimistic section (result trusted only after + * the version re-check) and under the bucket lock. + * 0 = hit (scratch filled), -2 = miss. + */ +static int scan_bucket(pcache_bucket_t *b, const str *key, unsigned int hash, + unsigned char tag, char *dst, unsigned int dstlen, + unsigned int *vlen_out, unsigned int *exp_out, + unsigned char *fl_out) +{ + unsigned long m, lo, hi; + unsigned int bound, vlen, klen, avail; + pcache_rec_t *r; + int i; + + pcache_arena_extents(&lo, &hi); + + for (m = tag_matches(b, tag); m; m &= m - 1) { + i = __builtin_ctzl(m) >> 3; + r = b->slot[i]; + if (!r) + continue; + + /* 3.2 copy-out rules: validate before every use. A stale + * pointer fails one of these or the final version re-check; + * a mismatch on live data is just not-this-slot */ + if ((unsigned long)r < lo || + (unsigned long)r + PCACHE_REC_HDR > hi) + continue; + bound = pcache_cell_bound(r); + if (!bound || (unsigned long)r + bound > hi) + continue; + if (r->hash != hash) + continue; + klen = r->klen; + if (klen != (unsigned int)key->len || + PCACHE_REC_HDR + klen > bound) + continue; + if (memcmp(r->data, key->s, klen)) + continue; + + vlen = r->vlen; /* aligned 4-byte load */ + /* subtractive, never additive: bound >= PCACHE_REC_HDR + klen was + * checked above, while PCACHE_REC_HDR + klen + vlen would wrap for + * a torn vlen and skip the very clamp that bounds a doomed copy */ + avail = bound - PCACHE_REC_HDR - klen; + if (vlen > avail) + vlen = avail; + /* report the true length either way, so a caller whose buffer is + * too small learns what it would need. Trusted, like every other + * field here, only once the version re-check passes */ + *vlen_out = vlen; + *exp_out = r->expires; + *fl_out = r->rflags; + /* probe: the caller wants existence, length and expiry, not the + * bytes. The metadata above is already published, so a probe + * validates exactly what a read would - it just stops here, + * before the copy (and without touching the record's payload). */ + if (!dst) + return 0; + if (vlen > dstlen) + return PCACHE_E_TOOSMALL; /* nothing copied */ + memcpy(dst, r->data + klen, vlen); + return 0; + } + return -2; +} + +/* overflow lookup - records are stable under the overflow lock */ +static int ovf_fetch(pcache_htable_t *ht, const str *key, unsigned int hash, + char *dst, unsigned int dstlen, unsigned int *vlen_out, + unsigned int *exp_out, unsigned char *fl_out) +{ + struct povf *n; + int rc = -2; + + lock_get(&ht->ovf_lock); + for (n = ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)]; n; n = n->next) { + if (n->hash != hash || n->rec->klen != key->len || + memcmp(n->rec->data, key->s, key->len)) + continue; + *vlen_out = n->rec->vlen; + *exp_out = n->rec->expires; + *fl_out = n->rec->rflags; + if (*vlen_out > dstlen) { + rc = PCACHE_E_TOOSMALL; /* nothing copied */ + break; + } + memcpy(dst, n->rec->data + key->len, *vlen_out); + rc = 0; + break; + } + lock_release(&ht->ovf_lock); + return rc; +} + +/* @now is a parameter (not read inside) so the selftest can run under a + * synthetic clock - get_ticks() is still 0 during mod_init */ +/* + * The one implementation of the read path. @dst/@dstlen is where the value + * lands: the internal scratch when the caller wanted an allocated str, or + * the caller's own buffer for the allocation-free entry point. Everything + * else - the optimistic loop, the lock fallback, the re-route retry, the + * overflow leg, expiry - is shared, so the two entry points can never + * disagree about which record they return. + */ +static int _pcache_ht_fetch_buf(pcache_htable_t *ht, const str *key, + char *dst, unsigned int dstlen, unsigned int *vlen_out, + unsigned int now, unsigned int *exp_out, long long *ll_out) +{ + pcache_bucket_t *b; + uint64_t route; + unsigned int hash, idx, v1, v2, vlen = 0, exp = 0, tries; + unsigned char tag, fl = 0; + long long ll; + int rc; + + /* fail closed: a caller that ignores the return code must not read an + * uninitialised length against a perfectly valid buffer */ + *vlen_out = 0; + if (ll_out) + *ll_out = 0; + if (exp_out) + *exp_out = 0; + + /* the destination now comes from outside the module on one of the + * entry points, so a bad one is an API misuse to reject, not an + * invariant to assume. dst == NULL with dstlen == 0 is the probe: + * existence and metadata, no copy (see pcache_ht_probe). */ + if (!ht || !key || (!dst && dstlen)) + return -1; + + hash = core_hash(key, NULL, 0); + tag = tag_of(hash); + +again: + idx = route_idx(ht, hash, &route); + b = bucket_at(ht, idx); + + rc = -2; + for (tries = 0; tries < PCACHE_SEQ_RETRIES; tries++) { + v1 = __atomic_load_n(&b->version, __ATOMIC_ACQUIRE); + if (v1 & 1) { + pcache_pause(); + continue; + } + rc = scan_bucket(b, key, hash, tag, dst, dstlen, &vlen, &exp, &fl); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + v2 = __atomic_load_n(&b->version, __ATOMIC_RELAXED); + if (v1 == v2) + goto settled; + } + + /* a writer is stalled mid-update: do not keep spinning - sleep on + * the lock (3.2 fallback; the lock sleeps under futex) */ + lock_get(&b->lock); + bkt_set_owner(b); + rc = scan_bucket(b, key, hash, tag, dst, dstlen, &vlen, &exp, &fl); + bkt_clear_owner(b); + lock_release(&b->lock); + HT_ST(ht, fallbacks); + +settled: + if (tries) + HT_ST_ADD(ht, retries, tries); + if (rc == -2) { + /* 3.4: a completed split may have re-routed the key */ + if (ht->route != route) + goto again; + if (ht->ovf_count) + rc = ovf_fetch(ht, key, hash, dst, dstlen, &vlen, &exp, &fl); + } + if (rc == -2) { + HT_ST(ht, misses); + return -2; + } + + if (exp && exp <= now) { + HT_ST(ht, misses); + return -2; /* expired-as-absent (3.5) */ + } + HT_ST(ht, hits); + if (exp_out) + *exp_out = exp; /* absolute ticks, 0 = never */ + *vlen_out = vlen; + + /* the value did not fit: the length above tells the caller what it + * would need, and @dst holds nothing usable */ + if (rc == PCACHE_E_TOOSMALL) + return PCACHE_E_TOOSMALL; + + if ((fl & PCACHE_F_INT) && vlen == 8) { + /* native counter: the 8 raw bytes are meaningless to the caller, + * so hand back the integer and let the entry point format it. + * A probe copied nothing, so there is no integer to read - it + * reports the hit and its metadata like any other record. */ + if (!dst) + return 1; + if (!ll_out) + return -1; + memcpy(&ll, dst, 8); + *ll_out = ll; + return 1; /* hit, and it is a counter */ + } + return 0; +} + +/* + * Allocating entry point: copies into the per-process scratch, then into a + * pkg buffer the caller owns. The scratch is PCACHE_CELL_MAX bytes, i.e. + * as large as the biggest legal record, so the too-small path below cannot + * be reached from here - it is handled defensively all the same. + */ +static int _pcache_ht_fetch(pcache_htable_t *ht, const str *key, str *val, + unsigned int now, unsigned int *exp_out) +{ + unsigned int vlen = 0; + long long ll = 0; + char *scratch; + int rc; + + scratch = get_scratch(); + if (!scratch) + return -1; + + rc = _pcache_ht_fetch_buf(ht, key, scratch, PCACHE_CELL_MAX, &vlen, + now, exp_out, &ll); + if (rc < 0) + return rc == PCACHE_E_TOOSMALL ? -1 : rc; + + if (rc == 1) { /* native counter: format on read */ + val->s = pkg_malloc(24); + if (!val->s) { + LM_ERR("no more pkg memory\n"); + return -1; + } + val->len = snprintf(val->s, 24, "%lld", ll); + return 0; + } + + val->s = pkg_malloc(vlen ? vlen : 1); + if (!val->s) { + LM_ERR("no more pkg memory for a %u byte value\n", vlen); + return -1; + } + memcpy(val->s, scratch, vlen); + val->len = vlen; + return 0; +} + +int pcache_ht_fetch(pcache_htable_t *ht, const str *key, str *val) +{ + return _pcache_ht_fetch(ht, key, val, get_ticks(), NULL); +} + +/* like pcache_ht_fetch, but also returns the record's absolute expiry + * (0 = never) - the MI perf_get needs the TTL alongside the value */ +int pcache_ht_fetch_ex(pcache_htable_t *ht, const str *key, str *val, + unsigned int *expires) +{ + return _pcache_ht_fetch(ht, key, val, get_ticks(), expires); +} + +/* existence probe - see the contract in pcache_htable.h. Shares the whole + * read path with the fetches (optimistic loop, lock fallback, re-route + * retry, overflow leg, expiry), stopping before the copy-out, so a probe + * can never disagree with a read about whether a key is there. */ +int pcache_ht_probe(pcache_htable_t *ht, const str *key, unsigned int *vlen, + unsigned int *expires, int *is_counter) +{ + unsigned int len = 0, exp = 0; + int rc; + + if (vlen) + *vlen = 0; + if (expires) + *expires = 0; + if (is_counter) + *is_counter = 0; + + rc = _pcache_ht_fetch_buf(ht, key, NULL, 0, &len, get_ticks(), + &exp, NULL); + if (rc < 0) + return rc; /* -2 = absent or expired */ + if (vlen) + *vlen = len; + if (expires) + *expires = exp; + /* the shared core reports a native counter as 1; a probe has nothing + * to hand back for one, but a caller may need to know it is not a + * plain value (a counter's meaning is local to the node holding it) */ + if (is_counter) + *is_counter = (rc == 1); + return 0; +} + +/* allocation-free entry point - see the contract in pcache_htable.h */ +int pcache_ht_fetch_buf(pcache_htable_t *ht, const str *key, char *buf, + unsigned int buflen, unsigned int *vlen, unsigned int *needed) +{ + long long ll = 0; + int rc; + + if (vlen) + *vlen = 0; + if (needed) + *needed = 0; + if (!vlen || !buf || buflen < PCACHE_GETBUF_MIN) { + LM_BUG("get_buf called with buf=%p buflen=%u vlen=%p\n", + buf, buflen, vlen); + return -1; + } + + rc = _pcache_ht_fetch_buf(ht, key, buf, buflen, vlen, get_ticks(), + NULL, &ll); + + if (rc == PCACHE_E_TOOSMALL) { + /* *vlen must never exceed the caller's buffer: {buf,*vlen} has to + * stay a valid str whatever the caller does with the return code */ + if (needed) + *needed = *vlen; + *vlen = 0; + return PCACHE_E_TOOSMALL; + } + if (rc < 0) + return rc; + if (rc == 1) /* counter: format into the caller's + * buffer; >= PCACHE_GETBUF_MIN is + * enforced above, so it always fits */ + *vlen = snprintf(buf, buflen, "%lld", ll); + return 0; +} + +/* writer-side slot scan, under the bucket lock - plain and exact */ +static int find_slot(pcache_bucket_t *b, const str *key, unsigned int hash, + unsigned char tag) +{ + pcache_rec_t *r; + unsigned int used = bkt_used(b), i; + + for (i = 0; i < used; i++) { + r = b->slot[i]; + if (b->tags[i] == tag && r && r->hash == hash && + r->klen == key->len && + !memcmp(r->data, key->s, key->len)) + return (int)i; + } + return -1; +} + +/* overflow search, under the overflow lock */ +static struct povf *ovf_find(pcache_htable_t *ht, const str *key, + unsigned int hash, struct povf ***prev_out) +{ + struct povf **prev = &ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)], *n; + + for (n = *prev; n; prev = &n->next, n = n->next) + if (n->hash == hash && n->rec->klen == key->len && + !memcmp(n->rec->data, key->s, key->len)) + break; + if (prev_out) + *prev_out = prev; + return n; +} + +int pcache_ht_store(pcache_htable_t *ht, const str *key, const str *val, + unsigned int expires) +{ + pcache_bucket_t *b; + pcache_rec_t *nr, *old = NULL; + struct povf *node = NULL, *on; + uint64_t route; + unsigned int hash, idx, used; + unsigned char tag; + int i, inserted = 0; + + if (key->len > 0xFFFF || + PCACHE_REC_SIZE(key->len, val->len) > PCACHE_CELL_MAX) { + PCACHE_REJECT_LOG("key %d + value %d bytes exceed the %d byte record limit\n", + key->len, val->len, PCACHE_CELL_MAX); + return -1; + } + + hash = core_hash(key, NULL, 0); + tag = tag_of(hash); + + /* build the full replacement record before any lock (3.5b rule 3) */ + nr = pcache_cell_alloc(PCACHE_REC_SIZE(key->len, val->len)); + if (!nr) + return -2; /* arena full - write dropped */ + nr->rflags = 0; + nr->klen = (unsigned short)key->len; + nr->vlen = (unsigned int)val->len; + nr->expires = expires; + nr->hash = hash; + memcpy(nr->data, key->s, key->len); + memcpy(nr->data + key->len, val->s, val->len); + +again: + idx = route_idx(ht, hash, &route); + b = bucket_at(ht, idx); + + lock_get(&b->lock); + bkt_set_owner(b); + + /* 3.4 writer rule: routing may have moved while we waited */ + if (ht->route != route) { + bkt_clear_owner(b); + lock_release(&b->lock); + goto again; + } + + i = find_slot(b, key, hash, tag); + if (i >= 0) { + old = b->slot[i]; + + if (old->vlen == (unsigned int)val->len && + !memcmp(old->data + key->len, val->s, val->len)) { + /* versionless TTL bump (2.7): the only mutation is one + * aligned store readers cannot tear - no version bumps, + * no reader disturbance */ + __atomic_store_n(&old->expires, expires, __ATOMIC_RELAXED); + hint_update(ht, idx, expires); + old = nr; /* discard the prebuilt one */ + goto done; + } + + if (PCACHE_REC_SIZE(key->len, val->len) <= pcache_cell_bound(old)) { + /* in-place: the new value fits the cell. + * + * Every seqlock ENTRY bump is ACQ_REL, not RELEASE. A + * release RMW only orders accesses that PRECEDE it and + * lets stores that follow be observed first, so on a + * weakly-ordered CPU (aarch64, ppc64le) the payload + * writes below could become visible before the version + * turned odd - a reader would then see an even version, + * scan a half-written record, re-read the same even + * version and accept the tear. The acquire half stops + * the hoist. (Linux writes seq++ then smp_wmb() for the + * same reason.) x86-64 is unaffected either way: the + * RMW is already a full barrier. EXIT bumps only need + * the release half, but are kept ACQ_REL so no site has + * to be classified by hand. */ + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + /* this is a plain value: drop any flag the cell carried + * from a previous life. Without it, storing an 8-byte + * string over a native counter left PCACHE_F_INT set and + * the read path re-interpreted the ASCII as an int64 */ + old->rflags = 0; + old->vlen = (unsigned int)val->len; + memcpy(old->data + key->len, val->s, val->len); + old->expires = expires; + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + old = nr; /* discard the prebuilt one */ + goto done; + } + + /* replace the record; the tag stays (same key, same hash) */ + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + b->slot[i] = nr; + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + goto done; + } + + /* not in the bucket - it may sit in overflow (uniqueness: a key is + * in its bucket or in overflow, never both) */ + if (ht->ovf_count) { + lock_get(&ht->ovf_lock); + on = ovf_find(ht, key, hash, NULL); + if (on) { + old = on->rec; + on->rec = nr; /* overflow readers are lock-serialized */ + lock_release(&ht->ovf_lock); + goto done; + } + lock_release(&ht->ovf_lock); + } + + used = bkt_used(b); + if (used < PCACHE_SLOTS) { + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + b->slot[used] = nr; + b->tags[used] = tag; + bkt_set_used(b, used + 1); + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + inserted = 1; + goto done; + } + + /* bucket full -> overflow. The chain node must not be allocated + * under the bucket lock, so drop it, allocate, re-take, re-check */ + if (!node) { + bkt_clear_owner(b); + lock_release(&b->lock); + node = pcache_cell_alloc(sizeof *node); + if (!node) { + pcache_cell_free(nr); + return -2; /* arena full - write dropped */ + } + goto again; + } + + lock_get(&ht->ovf_lock); + node->rec = nr; + node->hash = hash; + node->next = ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)]; + ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)] = node; + __atomic_add_fetch(&ht->ovf_count, 1, __ATOMIC_RELAXED); + lock_release(&ht->ovf_lock); + node = NULL; + inserted = 1; + +done: + bkt_clear_owner(b); + lock_release(&b->lock); + + /* frees strictly after the locks (3.5b) */ + if (old) + pcache_cell_free(old); + if (node) + pcache_cell_free(node); + HT_ST(ht, stores); + if (inserted) + HT_ST(ht, created); + return 0; +} + +int pcache_ht_add(pcache_htable_t *ht, const str *key, long long delta, + unsigned int expires, long long *new_val) +{ + pcache_bucket_t *b; + pcache_rec_t *nr, *r, *old = NULL; + struct povf *node = NULL, *on; + uint64_t route; + unsigned int hash, idx, used; + unsigned char tag; + long long cur; + int i, inserted = 0; + + if (key->len > 0xFFFF) + return -1; + + hash = core_hash(key, NULL, 0); + tag = tag_of(hash); + + /* the counter record is pre-built outside any lock (3.5b); it either + * becomes the entry (absent key / string conversion) or is freed */ + nr = pcache_cell_alloc(PCACHE_REC_SIZE(key->len, 8)); + if (!nr) + return -1; + nr->rflags = PCACHE_F_INT; + nr->klen = (unsigned short)key->len; + nr->vlen = 8; + nr->expires = expires; + nr->hash = hash; + memcpy(nr->data, key->s, key->len); + memcpy(nr->data + key->len, &delta, 8); + cur = delta; + +again: + idx = route_idx(ht, hash, &route); + b = bucket_at(ht, idx); + + lock_get(&b->lock); + bkt_set_owner(b); + + if (ht->route != route) { + bkt_clear_owner(b); + lock_release(&b->lock); + goto again; + } + + i = find_slot(b, key, hash, tag); + if (i >= 0) { + r = b->slot[i]; + if (r->rflags & PCACHE_F_INT) { + /* fixed-width accumulate; the payload may be unaligned, so + * it changes under the version, never bare */ + memcpy(&cur, r->data + r->klen, 8); + cur += delta; + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + memcpy(r->data + r->klen, &cur, 8); + r->expires = expires; + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + old = nr; + goto done; + } + /* string record: convert on first touch if numeric */ + if (pcache_str2ll(r->data + r->klen, r->vlen, &cur) < 0) + goto nan; + cur += delta; + memcpy(nr->data + key->len, &cur, 8); + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + b->slot[i] = nr; + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + old = r; + goto done; + } + + if (ht->ovf_count) { + lock_get(&ht->ovf_lock); + on = ovf_find(ht, key, hash, NULL); + if (on) { + r = on->rec; + if (r->rflags & PCACHE_F_INT) { + memcpy(&cur, r->data + r->klen, 8); + cur += delta; + memcpy(r->data + r->klen, &cur, 8); + r->expires = expires; + lock_release(&ht->ovf_lock); + old = nr; + goto done; + } + if (pcache_str2ll(r->data + r->klen, r->vlen, &cur) < 0) { + lock_release(&ht->ovf_lock); + goto nan; + } + cur += delta; + memcpy(nr->data + key->len, &cur, 8); + on->rec = nr; + lock_release(&ht->ovf_lock); + old = r; + goto done; + } + lock_release(&ht->ovf_lock); + } + + /* absent: nr already carries the delta as the initial value */ + used = bkt_used(b); + if (used < PCACHE_SLOTS) { + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + b->slot[used] = nr; + b->tags[used] = tag; + bkt_set_used(b, used + 1); + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + hint_update(ht, idx, expires); + inserted = 1; + goto done; + } + + if (!node) { + bkt_clear_owner(b); + lock_release(&b->lock); + node = pcache_cell_alloc(sizeof *node); + if (!node) { + pcache_cell_free(nr); + return -1; + } + goto again; + } + + lock_get(&ht->ovf_lock); + node->rec = nr; + node->hash = hash; + node->next = ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)]; + ht->ovf_tab[hash & (PCACHE_OVF_BUCKETS - 1)] = node; + __atomic_add_fetch(&ht->ovf_count, 1, __ATOMIC_RELAXED); + lock_release(&ht->ovf_lock); + node = NULL; + inserted = 1; + +done: + bkt_clear_owner(b); + lock_release(&b->lock); + + if (old) + pcache_cell_free(old); + if (node) + pcache_cell_free(node); + HT_ST(ht, stores); + if (inserted) + HT_ST(ht, created); + if (new_val) + *new_val = cur; + return 0; + +nan: + bkt_clear_owner(b); + lock_release(&b->lock); + PCACHE_REJECT_LOG("value of <%.*s> is not an integer\n", key->len, key->s); + pcache_cell_free(nr); + if (node) + pcache_cell_free(node); + return -1; +} + +/* + * Re-arm an existing key's TTL without touching its value (MI perf_ttl). + * The only mutation is one aligned store of `expires` under the bucket lock + * - the versionless TTL bump of 2.7: no version bump, no memcpy, so the + * lock-free readers are undisturbed and a reader that catches the store + * mid-flight sees either the old or the new value, never a torn one. + * @expires is absolute ticks (0 = never). 1 = re-armed, 0 = no such key. + */ +int pcache_ht_touch(pcache_htable_t *ht, const str *key, unsigned int expires) +{ + pcache_bucket_t *b; + pcache_rec_t *r; + struct povf *on; + uint64_t route; + unsigned int hash, idx; + unsigned char tag; + int i, rc = 0; + + hash = core_hash(key, NULL, 0); + tag = tag_of(hash); + +again: + idx = route_idx(ht, hash, &route); + b = bucket_at(ht, idx); + + lock_get(&b->lock); + bkt_set_owner(b); + if (ht->route != route) { /* 3.4 writer rule: re-routed */ + bkt_clear_owner(b); + lock_release(&b->lock); + goto again; + } + + i = find_slot(b, key, hash, tag); + if (i >= 0) { + r = b->slot[i]; + __atomic_store_n(&r->expires, expires, __ATOMIC_RELAXED); + hint_update(ht, idx, expires); + rc = 1; + } + bkt_clear_owner(b); + lock_release(&b->lock); + + if (rc) + return 1; + + /* a bucket miss may mean a completed split re-routed the key */ + if (ht->route != route) + goto again; + + /* else it may sit in overflow (hash-keyed, under the overflow lock) */ + if (ht->ovf_count) { + lock_get(&ht->ovf_lock); + on = ovf_find(ht, key, hash, NULL); + if (on) { + on->rec->expires = expires; + rc = 1; + } + lock_release(&ht->ovf_lock); + } + return rc; +} + +int pcache_ht_remove(pcache_htable_t *ht, const str *key) +{ + pcache_bucket_t *b; + pcache_rec_t *dead = NULL; + struct povf *on = NULL, **prev; + uint64_t route; + unsigned int hash, idx, used; + unsigned char tag; + int i; + + hash = core_hash(key, NULL, 0); + tag = tag_of(hash); + +again: + idx = route_idx(ht, hash, &route); + b = bucket_at(ht, idx); + + lock_get(&b->lock); + bkt_set_owner(b); + + if (ht->route != route) { + bkt_clear_owner(b); + lock_release(&b->lock); + goto again; + } + + i = find_slot(b, key, hash, tag); + if (i >= 0) { + dead = b->slot[i]; + used = bkt_used(b); + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + b->slot[i] = b->slot[used - 1]; /* compact: readers retry */ + b->tags[i] = b->tags[used - 1]; + b->slot[used - 1] = NULL; + b->tags[used - 1] = 0; + bkt_set_used(b, used - 1); + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + } else if (ht->ovf_count) { + lock_get(&ht->ovf_lock); + on = ovf_find(ht, key, hash, &prev); + if (on) { + *prev = on->next; + __atomic_sub_fetch(&ht->ovf_count, 1, __ATOMIC_RELAXED); + dead = on->rec; + } + lock_release(&ht->ovf_lock); + } + + bkt_clear_owner(b); + lock_release(&b->lock); + + if (dead) { + pcache_cell_free(dead); + HT_ST(ht, removes); + HT_ST(ht, destroyed); + } + if (on) + pcache_cell_free(on); + return dead ? 1 : 0; +} + +/* + * One optimistic seqlock snapshot of slot @i of bucket @b into @kbuf/@vbuf + * (each >= PCACHE_CELL_MAX), applying the 3.2 copy-out clamps and the + * stalled-writer lock fallback. Returns 1 and fills the out-params if a + * live record was captured, 0 if the slot is empty. Shared verbatim by the + * full-table walk (pcache_ht_iter) and the cursored scan (pcache_ht_scan). + */ +static int snapshot_slot(pcache_bucket_t *b, unsigned int i, + char *kbuf, char *vbuf, unsigned int *klen_o, unsigned int *vlen_o, + unsigned int *exp_o, unsigned char *fl_o, + unsigned long lo, unsigned long hi) +{ + pcache_rec_t *r; + unsigned int v1, v2, tries, bound = 0, klen = 0, vlen = 0, exp = 0; + unsigned char fl = 0; + int have = 0; + + for (tries = 0; tries < PCACHE_SEQ_RETRIES; tries++) { + v1 = __atomic_load_n(&b->version, __ATOMIC_ACQUIRE); + if (v1 & 1) { + pcache_pause(); + continue; + } + r = b->slot[i]; + /* 3.2 copy-out rules, as in scan_bucket() */ + if (r && ((unsigned long)r < lo || + (unsigned long)r + PCACHE_REC_HDR > hi)) + r = NULL; + if (r) { + bound = pcache_cell_bound(r); + if (!bound || (unsigned long)r + bound > hi) + r = NULL; + } + if (r) { + klen = r->klen; + if (PCACHE_REC_HDR + klen > bound) + klen = bound - PCACHE_REC_HDR; + vlen = r->vlen; + if (PCACHE_REC_HDR + klen + vlen > bound) + vlen = bound - PCACHE_REC_HDR - klen; /* see scan_bucket */ + memcpy(kbuf, r->data, klen); + memcpy(vbuf, r->data + klen, vlen); + exp = r->expires; + fl = r->rflags; + } + __atomic_thread_fence(__ATOMIC_ACQUIRE); + v2 = __atomic_load_n(&b->version, __ATOMIC_RELAXED); + if (v1 == v2) { + have = r != NULL; + break; + } + } + if (tries == PCACHE_SEQ_RETRIES) { + /* stalled writer: read this slot under the lock (record stable) */ + lock_get(&b->lock); + bkt_set_owner(b); + r = b->slot[i]; + if (r) { + klen = r->klen; + vlen = r->vlen; + exp = r->expires; + fl = r->rflags; + memcpy(kbuf, r->data, klen); + memcpy(vbuf, r->data + klen, vlen); + have = 1; + } + bkt_clear_owner(b); + lock_release(&b->lock); + } + if (!have) + return 0; + *klen_o = klen; *vlen_o = vlen; *exp_o = exp; *fl_o = fl; + return 1; +} + +/* + * Format one snapshotted entry (native counters -> decimal), NUL-terminate + * both buffers and hand it to the callback. @vbuf must have room for the + * 24-byte decimal form. Returns the callback's rc (<0 stops the walk). + */ +static int emit_entry(pcache_iter_cb cb, void *ctx, char *kbuf, + unsigned int klen, char *vbuf, unsigned int vlen, + unsigned int exp, unsigned char fl) +{ + str key, val; + long long ll; + + if ((fl & PCACHE_F_INT) && vlen == 8) { + memcpy(&ll, vbuf, 8); + vlen = snprintf(vbuf, 24, "%lld", ll); + } + kbuf[klen] = 0; + vbuf[vlen] = 0; + key.s = kbuf; key.len = klen; + val.s = vbuf; val.len = vlen; + return cb(&key, &val, exp, ctx); +} + +/* walk the overflow leg under the overflow lock; @kbuf/@vbuf are the caller's + * snapshot buffers. Returns the last callback rc (<0 stops). */ +static int iter_overflow(pcache_htable_t *ht, pcache_iter_cb cb, void *ctx, + char *kbuf, char *vbuf) +{ + pcache_rec_t *r; + struct povf *n; + unsigned int idx, klen, vlen; + int rc = 0; + + if (!ht->ovf_count) + return 0; + lock_get(&ht->ovf_lock); + for (idx = 0; idx < PCACHE_OVF_BUCKETS && rc >= 0; idx++) { + for (n = ht->ovf_tab[idx]; n; n = n->next) { + r = n->rec; + klen = r->klen; + vlen = r->vlen; + memcpy(kbuf, r->data, klen); + memcpy(vbuf, r->data + klen, vlen); + rc = emit_entry(cb, ctx, kbuf, klen, vbuf, vlen, + r->expires, r->rflags); + if (rc < 0) + break; + } + } + lock_release(&ht->ovf_lock); + return rc; +} + +int pcache_ht_iter(pcache_htable_t *ht, pcache_iter_cb cb, void *ctx) +{ + pcache_bucket_t *b; + unsigned long lo, hi; + unsigned int idx, i, klen, vlen, exp; + unsigned char fl; + char *kbuf, *vbuf; + int rc = 0; + + kbuf = pkg_malloc(2 * PCACHE_CELL_MAX); + if (!kbuf) { + LM_ERR("no more pkg memory for the walk buffers\n"); + return -1; + } + vbuf = kbuf + PCACHE_CELL_MAX; + + pcache_arena_extents(&lo, &hi); + + for (idx = 0; idx < ht->nbuckets; idx++) { + b = bucket_at(ht, idx); + for (i = 0; i < PCACHE_SLOTS; i++) { + if (!snapshot_slot(b, i, kbuf, vbuf, &klen, &vlen, + &exp, &fl, lo, hi)) + continue; + rc = emit_entry(cb, ctx, kbuf, klen, vbuf, vlen, exp, fl); + if (rc < 0) + goto out; + } + } + + /* overflow leg - under the lock; the callback must not re-enter */ + rc = iter_overflow(ht, cb, ctx, kbuf, vbuf); +out: + pkg_free(kbuf); + return rc < 0 ? rc : 0; +} + +/* + * Cursored, bounded walk for the MI perf_scan (Redis SCAN semantics). From + * bucket *@cursor it visits up to @max_buckets buckets, invoking @cb per live + * entry, then sets *@cursor to the bucket to resume from - or 0 once the walk + * is complete, the overflow leg being emitted in that final call. Buckets + * never move and the table only grows (3.4), so a plain ascending cursor gives + * the >=-once guarantee and stays valid across a concurrent resize. The cursor + * advances a whole bucket at a time, so @cb sees every entry of a visited + * bucket and is never asked to stop mid-bucket (no intra-bucket duplicates on + * resume). Returns 0, or <0 on error / callback stop. + */ +int pcache_ht_scan(pcache_htable_t *ht, unsigned int *cursor, + unsigned int max_buckets, pcache_iter_cb cb, void *ctx) +{ + pcache_bucket_t *b; + unsigned long lo, hi; + unsigned int idx, end, i, klen, vlen, exp, nb; + unsigned char fl; + char *kbuf, *vbuf; + int rc = 0; + + if (!max_buckets) + max_buckets = PCACHE_SCAN_BUCKETS; + + kbuf = pkg_malloc(2 * PCACHE_CELL_MAX); + if (!kbuf) { + LM_ERR("no more pkg memory for the scan buffers\n"); + return -1; + } + vbuf = kbuf + PCACHE_CELL_MAX; + pcache_arena_extents(&lo, &hi); + + nb = ht->nbuckets; + idx = *cursor; + end = (idx > nb || nb - idx < max_buckets) ? nb : idx + max_buckets; + + for (; idx < end; idx++) { + b = bucket_at(ht, idx); + for (i = 0; i < PCACHE_SLOTS; i++) { + if (!snapshot_slot(b, i, kbuf, vbuf, &klen, &vlen, + &exp, &fl, lo, hi)) + continue; + rc = emit_entry(cb, ctx, kbuf, klen, vbuf, vlen, exp, fl); + if (rc < 0) + goto out; + } + } + + if (idx < nb) { + *cursor = idx; /* more buckets remain */ + goto out; + } + + /* last bucket reached: drain overflow once, then signal completion */ + rc = iter_overflow(ht, cb, ctx, kbuf, vbuf); + *cursor = 0; +out: + pkg_free(kbuf); + return rc < 0 ? rc : 0; +} + +unsigned int pcache_ht_nbuckets(pcache_htable_t *ht) +{ + return __atomic_load_n(&ht->nbuckets, __ATOMIC_RELAXED); +} + +unsigned int pcache_ht_sweep(pcache_htable_t *ht, unsigned int now, + pcache_expired_cb cb, void *cb_ctx) +{ + pcache_bucket_t *b; + pcache_rec_t *r, *dead[PCACHE_SLOTS]; + pcache_rec_t *batch_r[64]; + struct povf *n, **prev, *batch_n[64]; + unsigned int idx, i, used, hint, newmin, ndead, freed = 0; + str dk; + int bn; + + for (idx = 0; idx < ht->nbuckets; idx++) { + hint = *hint_at(ht, idx); + if (!hint || hint > now) + continue; /* 16 hints per line, no bucket touch */ + + b = bucket_at(ht, idx); + lock_get(&b->lock); + bkt_set_owner(b); + + ndead = 0; + newmin = 0; + i = 0; + while (i < (used = bkt_used(b))) { + r = b->slot[i]; + if (r->expires && r->expires <= now) { + if (!ndead) + __atomic_add_fetch(&b->version, 1, + __ATOMIC_ACQ_REL); + dead[ndead++] = r; + b->slot[i] = b->slot[used - 1]; + b->tags[i] = b->tags[used - 1]; + b->slot[used - 1] = NULL; + b->tags[used - 1] = 0; + bkt_set_used(b, used - 1); + continue; /* re-examine the swapped-in slot */ + } + if (r->expires && (!newmin || r->expires < newmin)) + newmin = r->expires; + i++; + } + if (ndead) + __atomic_add_fetch(&b->version, 1, __ATOMIC_ACQ_REL); + *hint_at(ht, idx) = newmin; + + bkt_clear_owner(b); + lock_release(&b->lock); + + /* reclamation strictly after the lock (3.5b), through the + * global pool - the sweeping process is not an allocator */ + for (i = 0; i < ndead; i++) { + if (cb) { + dk.s = dead[i]->data; + dk.len = dead[i]->klen; + cb(&dk, cb_ctx); /* CP-11 expiry event, unlocked */ + } + pcache_cell_free_global(dead[i]); + } + freed += ndead; + } + + if (!ht->ovf_count) { + HT_ST_ADD(ht, destroyed, freed); + HT_ST_ADD(ht, expired, freed); + return freed; + } + + /* overflow: unhinted, scanned whole - it exists to be small */ + for (idx = 0; idx < PCACHE_OVF_BUCKETS; idx++) { + do { + bn = 0; + lock_get(&ht->ovf_lock); + prev = &ht->ovf_tab[idx]; + for (n = *prev; n && bn < 64; ) { + if (n->rec->expires && n->rec->expires <= now) { + *prev = n->next; + batch_n[bn] = n; + batch_r[bn] = n->rec; + bn++; + __atomic_sub_fetch(&ht->ovf_count, 1, + __ATOMIC_RELAXED); + n = *prev; + } else { + prev = &n->next; + n = n->next; + } + } + lock_release(&ht->ovf_lock); + for (i = 0; i < (unsigned int)bn; i++) { + if (cb) { + dk.s = batch_r[i]->data; + dk.len = batch_r[i]->klen; + cb(&dk, cb_ctx); /* CP-11 expiry event, unlocked */ + } + pcache_cell_free_global(batch_r[i]); + pcache_cell_free_global(batch_n[i]); + } + freed += bn; + } while (bn == 64); + } + + HT_ST_ADD(ht, destroyed, freed); + HT_ST_ADD(ht, expired, freed); + return freed; +} + +void pcache_ht_totals(pcache_htable_t *ht, pcache_ht_totals_t *out) +{ + pcache_pstat_t *p; + unsigned int i; + + memset(out, 0, sizeof *out); + for (i = 0; i < ht->pstats_n; i++) { + p = &ht->pstats[i]; + out->hits += p->hits; + out->misses += p->misses; + out->stores += p->stores; + out->removes += p->removes; + out->created += p->created; + out->destroyed += p->destroyed; + out->expired += p->expired; + out->retries += p->retries; + out->fallbacks += p->fallbacks; + } + /* live gauge: always absolute, never relative to a reset */ + out->entries = out->created - out->destroyed; + + /* everything else is a running total - report it since the last reset */ + out->hits -= ht->base.hits; + out->misses -= ht->base.misses; + out->stores -= ht->base.stores; + out->removes -= ht->base.removes; + out->created -= ht->base.created; + out->destroyed -= ht->base.destroyed; + out->expired -= ht->base.expired; + out->retries -= ht->base.retries; + out->fallbacks -= ht->base.fallbacks; +} + +void pcache_ht_stats_reset(pcache_htable_t *ht) +{ + pcache_ht_totals_t now; + unsigned long entries; + + /* read through the current baseline, then fold it back in: the shards + * are only ever read here, never rewound, so a worker incrementing one + * concurrently just lands in the next interval. */ + pcache_ht_totals(ht, &now); + entries = now.entries; + + ht->base.hits += now.hits; + ht->base.misses += now.misses; + ht->base.stores += now.stores; + ht->base.removes += now.removes; + ht->base.created += now.created; + ht->base.destroyed += now.destroyed; + ht->base.expired += now.expired; + ht->base.retries += now.retries; + ht->base.fallbacks += now.fallbacks; + + LM_INFO("statistics reset; %lu entries live\n", entries); +} + +/* + * CP-09: linear-hash growth. The maintenance timer is the SOLE splitter, so + * splits never race one another; readers and writers use the routing word + * plus the 3.4 re-check protocol already wired into fetch/store/remove. + * Existing buckets never move (growth appends), so no pointer invalidation. + */ + +/* allocate the segment (+ its hint segment) containing bucket @idx if absent. + * Single-splitter, so no alloc race; the seg pointer is published (release) + * only once fully built, and always before the routing word that makes any + * bucket in it reachable. */ +static int ensure_segment(pcache_htable_t *ht, unsigned int idx) +{ + unsigned int s = idx >> PCACHE_SEG_BITS, i; + pcache_bucket_t *seg; + unsigned int *hseg; + + if (ht->seg[s]) + return 0; + seg = pcache_region_alloc((unsigned long)PCACHE_SEG_SIZE * sizeof *seg); + if (!seg) + return -1; + memset(seg, 0, (unsigned long)PCACHE_SEG_SIZE * sizeof *seg); + for (i = 0; i < PCACHE_SEG_SIZE; i++) + lock_init(&seg[i].lock); + hseg = pcache_region_alloc(PCACHE_SEG_SIZE * sizeof(unsigned int)); + if (!hseg) + return -1; + memset(hseg, 0, PCACHE_SEG_SIZE * sizeof(unsigned int)); + ht->hint_seg[s] = hseg; + __atomic_store_n(&ht->seg[s], seg, __ATOMIC_RELEASE); + return 1; +} + +/* + * Split the current bucket (index = split), redistributing its 6 slots into + * itself and the new partner (split + 2^level) by bit `level` of each entry's + * stored hash (no rehash). Overflow is hash-keyed and bucket-agnostic + * (ovf_find matches by hash+key regardless of routing), so a split leaves + * overflow entries findable and does not touch them - they drain as the + * freed slots absorb new inserts. Returns 1 on a split, 0 at the ceiling, + * -1 on OOM. + */ +static int pcache_ht_split(pcache_htable_t *ht) +{ + uint64_t r = ht->route, nr; + unsigned int level = (unsigned int)(r >> 32); + unsigned int split = (unsigned int)r; + unsigned int sidx = split, pidx = split + (1U << level); + pcache_bucket_t *S, *P; + unsigned int used, pused, i, smin = 0, pmin = 0; + pcache_rec_t *rec; + + if (pidx >= PCACHE_NSEGS * PCACHE_SEG_SIZE) + return 0; /* at the 2^24 ceiling */ + if (ensure_segment(ht, pidx) < 0) + return -1; + + S = bucket_at(ht, sidx); + P = bucket_at(ht, pidx); /* fresh, zeroed, unreachable */ + + lock_get(&S->lock); + bkt_set_owner(S); + __atomic_add_fetch(&S->version, 1, __ATOMIC_ACQ_REL); /* writer in */ + + used = bkt_used(S); + pused = 0; + i = 0; + while (i < used) { + rec = S->slot[i]; + if ((rec->hash >> level) & 1) { /* -> partner */ + P->slot[pused] = rec; + P->tags[pused] = S->tags[i]; + pused++; + S->slot[i] = S->slot[used - 1]; + S->tags[i] = S->tags[used - 1]; + S->slot[used - 1] = NULL; + S->tags[used - 1] = 0; + used--; + } else { + i++; + } + } + bkt_set_used(S, used); + bkt_set_used(P, pused); + + /* recompute both expiry hints (moved entries left S) */ + for (i = 0; i < used; i++) + if (S->slot[i]->expires && (!smin || S->slot[i]->expires < smin)) + smin = S->slot[i]->expires; + for (i = 0; i < pused; i++) + if (P->slot[i]->expires && (!pmin || P->slot[i]->expires < pmin)) + pmin = P->slot[i]->expires; + *hint_at(ht, sidx) = smin; + *hint_at(ht, pidx) = pmin; + + /* Publish the new routing word WHILE S's version is odd. The even + * bump below is a release that happens-after this store, so any + * reader which later observes S even (via acquire) and misses a + * moved key is guaranteed to see the new route on its 3.4 re-read + * and re-route to the partner - no false-miss window. */ + if (split + 1 == (1U << level)) + nr = (uint64_t)(level + 1) << 32; /* level up, split 0 */ + else + nr = ((uint64_t)level << 32) | (split + 1); + __atomic_store_n(&ht->route, nr, __ATOMIC_RELEASE); + ht->nbuckets++; + + __atomic_add_fetch(&S->version, 1, __ATOMIC_ACQ_REL); /* S stable */ + bkt_clear_owner(S); + lock_release(&S->lock); + return 1; +} + +/* + * Split buckets while the load factor exceeds @target_lf, up to @budget + * splits. Called only from the maintenance timer (single splitter). The + * live-entry count is read once - splitting only redistributes, never + * changes it - so the loop just watches nbuckets climb. Returns the number + * of splits performed. + */ +unsigned int pcache_ht_grow(pcache_htable_t *ht, unsigned int target_lf, + unsigned int budget) +{ + pcache_ht_totals_t t; + unsigned int did = 0; + + if (!target_lf) + return 0; + pcache_ht_totals(ht, &t); + while (did < budget && + t.entries > (unsigned long)target_lf * ht->nbuckets) { + if (pcache_ht_split(ht) <= 0) + break; /* ceiling or OOM */ + did++; + } + return did; +} + +pcache_htable_t *pcache_htable_new(unsigned int size_log2) +{ + pcache_htable_t *ht; + pcache_bucket_t *seg; + unsigned int nbuckets = 1U << size_log2, done, n, s, i; + + ht = pcache_region_alloc(sizeof *ht); + if (!ht) + return NULL; + memset(ht, 0, sizeof *ht); + + /* Segments are FIXED at PCACHE_SEG_SIZE buckets (the directory is a + * directory of fixed segments, DESIGN 3.4) - always allocate full + * segments, even when the initial nbuckets is smaller, so linear-hash + * growth can fill a segment up to its boundary without going out of + * bounds. n rounds nbuckets up to whole segments. */ + n = (nbuckets + PCACHE_SEG_SIZE - 1) / PCACHE_SEG_SIZE; + if (n == 0) + n = 1; + for (s = 0; s < n; s++) { + seg = pcache_region_alloc( + (unsigned long)PCACHE_SEG_SIZE * sizeof *seg); + if (!seg) + return NULL; + memset(seg, 0, (unsigned long)PCACHE_SEG_SIZE * sizeof *seg); + for (i = 0; i < PCACHE_SEG_SIZE; i++) + lock_init(&seg[i].lock); + ht->seg[s] = seg; + + ht->hint_seg[s] = pcache_region_alloc( + PCACHE_SEG_SIZE * sizeof(unsigned int)); + if (!ht->hint_seg[s]) + return NULL; + memset(ht->hint_seg[s], 0, + PCACHE_SEG_SIZE * sizeof(unsigned int)); + } + (void)done; + + ht->pstats_n = PCACHE_MAX_PROCS; + ht->pstats = pcache_region_alloc( + (unsigned long)ht->pstats_n * sizeof *ht->pstats); + if (!ht->pstats) + return NULL; + memset(ht->pstats, 0, + (unsigned long)ht->pstats_n * sizeof *ht->pstats); + + ht->ovf_tab = pcache_region_alloc( + PCACHE_OVF_BUCKETS * sizeof *ht->ovf_tab); + if (!ht->ovf_tab) + return NULL; + memset(ht->ovf_tab, 0, PCACHE_OVF_BUCKETS * sizeof *ht->ovf_tab); + if (!lock_init(&ht->ovf_lock)) + return NULL; + + ht->nbuckets = nbuckets; + ht->route = (uint64_t)size_log2 << 32; + + LM_DBG("table ready: %u buckets in %u segments\n", nbuckets, s); + return ht; +} + + +/* + * startup selftest (modparam "htable_selftest"): single-process coverage + * of every path above - roundtrip, in-place vs replacement, the + * versionless bump (bucket version must NOT move), removal compaction, + * overflow spill and drain, expiry-as-absent, record-size limits. + * Multi-process interleavings are CP-16's job. + */ +#define HCHK(cond, ...) \ + do { \ + if (!(cond)) { \ + LM_ERR("htable selftest FAILED: " __VA_ARGS__); \ + return -1; \ + } \ + } while (0) + +struct st_walk { + unsigned char seen[200]; + unsigned int total, bad; +}; + +static int st_walk_cb(const str *key, const str *val, unsigned int exp, + void *ctx) +{ + struct st_walk *w = ctx; + unsigned int i; + char vb[32]; + + w->total++; + if (key->len != 9 || memcmp(key->s, "spill-", 6) != 0 || + sscanf(key->s + 6, "%u", &i) != 1 || i >= 200) { + w->bad++; + return 0; + } + w->seen[i]++; + snprintf(vb, sizeof vb, "payload-%03u", i); + if (val->len != strlen(vb) || memcmp(val->s, vb, val->len)) + w->bad++; + return 0; +} + +static pcache_rec_t *st_slot_of(pcache_htable_t *ht, const str *key) +{ + uint64_t route; + unsigned int hash = core_hash((str *)key, NULL, 0); + pcache_bucket_t *b = bucket_at(ht, route_idx(ht, hash, &route)); + int i = find_slot(b, key, hash, tag_of(hash)); + + return i < 0 ? NULL : b->slot[i]; +} + +int pcache_htable_selftest(void) +{ + pcache_htable_t *ht; + pcache_rec_t *r0, *r1; + pcache_bucket_t *b; + str k, v, out; + uint64_t route; + unsigned int i, ver0, ver1, nb_used; + char kb[32], vb[512]; + int rc; + + ht = pcache_htable_new(4); /* 16 buckets: collisions */ + HCHK(ht != NULL, "table creation failed\n"); + + /* roundtrip + miss */ + k.s = "key-one"; k.len = 7; + v.s = "value-one"; v.len = 9; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "store failed\n"); + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && out.len == 9 && !memcmp(out.s, "value-one", 9), + "roundtrip mismatch (rc %d)\n", rc); + pkg_free(out.s); + k.s = "absent"; k.len = 6; + HCHK(pcache_ht_fetch(ht, &k, &out) == -2, "phantom hit\n"); + + /* in-place overwrite: same cell, new bytes */ + k.s = "key-one"; k.len = 7; + r0 = st_slot_of(ht, &k); + HCHK(r0 != NULL, "stored key has no slot\n"); + v.s = "VALUE-two"; v.len = 9; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "overwrite failed\n"); + r1 = st_slot_of(ht, &k); + HCHK(r1 == r0, "same-size overwrite moved the record\n"); + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && !memcmp(out.s, "VALUE-two", 9), "overwrite lost\n"); + pkg_free(out.s); + + /* versionless TTL bump: byte-identical value, version must hold */ + b = bucket_at(ht, route_idx(ht, core_hash(&k, NULL, 0), &route)); + ver0 = b->version; + HCHK(pcache_ht_store(ht, &k, &v, get_ticks() + 100) == 0, + "bump store failed\n"); + ver1 = b->version; + HCHK(ver0 == ver1, "TTL bump bumped the version (%u -> %u)\n", + ver0, ver1); + HCHK(st_slot_of(ht, &k)->expires == get_ticks() + 100, + "TTL bump did not land\n"); + + /* replacement: value outgrows the cell class */ + memset(vb, 'R', sizeof vb); + v.s = vb; v.len = 300; /* 16+7+300 -> bigger class */ + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "grow store failed\n"); + r1 = st_slot_of(ht, &k); + HCHK(r1 != r0, "cross-class grow did not replace the record\n"); + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && out.len == 300 && out.s[0] == 'R' && out.s[299] == 'R', + "grown value mismatch\n"); + pkg_free(out.s); + + /* remove + idempotent remove */ + HCHK(pcache_ht_remove(ht, &k) == 1, "remove failed\n"); + HCHK(pcache_ht_fetch(ht, &k, &out) == -2, "removed key still hits\n"); + HCHK(pcache_ht_remove(ht, &k) == 0, "second remove not idempotent\n"); + + /* expiry-as-absent, under a synthetic clock (get_ticks() is still 0 + * in mod_init, so nothing can be "in the past" through the public + * wrapper here) */ + v.s = "temp"; v.len = 4; + HCHK(pcache_ht_store(ht, &k, &v, 500) == 0, "expired store failed\n"); + HCHK(_pcache_ht_fetch(ht, &k, &out, 1000, NULL) == -2, + "expired key still hits\n"); + rc = _pcache_ht_fetch(ht, &k, &out, 400, NULL); + HCHK(rc == 0, "live key missed\n"); + pkg_free(out.s); + pcache_ht_remove(ht, &k); + + /* native counters (CP-04): create, accumulate, format-on-read, + * string conversion, NaN refusal */ + { + long long nv = 0; + + k.s = "ctr"; k.len = 3; + HCHK(pcache_ht_add(ht, &k, 5, 0, &nv) == 0 && nv == 5, + "counter create: %lld\n", nv); + HCHK(pcache_ht_add(ht, &k, 37, 0, &nv) == 0 && nv == 42, + "counter accumulate: %lld\n", nv); + HCHK(pcache_ht_add(ht, &k, -2, 0, &nv) == 0 && nv == 40, + "counter subtract: %lld\n", nv); + r0 = st_slot_of(ht, &k); + HCHK(r0 && (r0->rflags & PCACHE_F_INT), "counter not native\n"); + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && out.len == 2 && !memcmp(out.s, "40", 2), + "counter fetch not formatted: <%.*s>\n", out.len, out.s); + pkg_free(out.s); + HCHK(pcache_ht_remove(ht, &k) == 1, "counter remove\n"); + + /* a numeric string converts on the first add */ + k.s = "s2c"; k.len = 3; + v.s = "100"; v.len = 3; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "s2c store\n"); + HCHK(pcache_ht_add(ht, &k, 1, 0, &nv) == 0 && nv == 101, + "s2c add: %lld\n", nv); + r0 = st_slot_of(ht, &k); + HCHK(r0 && (r0->rflags & PCACHE_F_INT), "s2c not converted\n"); + HCHK(pcache_ht_remove(ht, &k) == 1, "s2c remove\n"); + + /* a non-numeric string refuses */ + k.s = "nan"; k.len = 3; + v.s = "abc"; v.len = 3; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "nan store\n"); + /* drives the reject path on purpose - see st_expect_reject */ + st_expect_reject = 1; + rc = pcache_ht_add(ht, &k, 1, 0, &nv); + st_expect_reject = 0; + HCHK(rc == -1, "nan add passed\n"); + HCHK(pcache_ht_remove(ht, &k) == 1, "nan remove\n"); + } + + /* record-size limit */ + k.s = "key-one"; k.len = 7; + v.s = vb; v.len = PCACHE_CELL_MAX; /* header pushes it over */ + /* an expected rejection too - see st_expect_reject */ + st_expect_reject = 1; + rc = pcache_ht_store(ht, &k, &v, 0); + st_expect_reject = 0; + HCHK(rc == -1, "oversize store passed\n"); + + /* overflow: 200 keys over 16 buckets force chains, then drain */ + for (i = 0; i < 200; i++) { + k.len = snprintf(kb, sizeof kb, "spill-%03u", i); k.s = kb; + v.len = snprintf(vb, sizeof vb, "payload-%03u", i); v.s = vb; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "spill store %u\n", i); + } + HCHK(ht->ovf_count > 0, "200 keys over 16 buckets never overflowed\n"); + LM_INFO("htable selftest: %u of 200 keys in overflow\n", ht->ovf_count); + for (i = 0; i < 200; i++) { + k.len = snprintf(kb, sizeof kb, "spill-%03u", i); k.s = kb; + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0, "spill fetch %u missed (rc %d)\n", i, rc); + v.len = snprintf(vb, sizeof vb, "payload-%03u", i); + HCHK(out.len == (unsigned int)v.len && !memcmp(out.s, vb, v.len), + "spill value %u mismatch\n", i); + pkg_free(out.s); + } + /* walker: exactly-once coverage of bucket + overflow legs (single + * process, so deterministic), values verified in the callback */ + { + struct st_walk w; + memset(&w, 0, sizeof w); + HCHK(pcache_ht_iter(ht, st_walk_cb, &w) == 0, "walk failed\n"); + HCHK(w.total == 200 && w.bad == 0, + "walk saw %u entries, %u bad\n", w.total, w.bad); + for (i = 0; i < 200; i++) + HCHK(w.seen[i] == 1, "walk saw key %u %u times\n", + i, w.seen[i]); + } + + /* overwrite one overflow resident, verify, then drain everything */ + k.len = snprintf(kb, sizeof kb, "spill-%03u", 199); k.s = kb; + v.s = "moved"; v.len = 5; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "ovf overwrite failed\n"); + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && out.len == 5 && !memcmp(out.s, "moved", 5), + "ovf overwrite lost\n"); + pkg_free(out.s); + for (i = 0; i < 200; i++) { + k.len = snprintf(kb, sizeof kb, "spill-%03u", i); k.s = kb; + HCHK(pcache_ht_remove(ht, &k) == 1, "spill remove %u\n", i); + } + HCHK(ht->ovf_count == 0, "overflow not drained: %u left\n", + ht->ovf_count); + for (i = 0, nb_used = 0; i < ht->nbuckets; i++) + nb_used += bkt_used(bucket_at(ht, i)); + HCHK(nb_used == 0, "%u slots still used after full drain\n", nb_used); + + /* expiry sweep (CP-05): hint-routed, bucket + overflow legs, mixed + * with never-expiring survivors */ + { + unsigned int freed; + + /* the never-expiring survivor goes in FIRST so it takes a bucket + * slot - stored last it would land in overflow and the count + * checks below would misread a correct sweep */ + k.s = "stay"; k.len = 4; + v.s = "keep"; v.len = 4; + HCHK(pcache_ht_store(ht, &k, &v, 0) == 0, "stay store\n"); + for (i = 0; i < 100; i++) { + k.len = snprintf(kb, sizeof kb, "ex-%03u", i); k.s = kb; + v.s = "tmp"; v.len = 3; + HCHK(pcache_ht_store(ht, &k, &v, 10) == 0, + "sweep store %u\n", i); + } + HCHK(ht->ovf_count > 0, "sweep set never overflowed\n"); + + freed = pcache_ht_sweep(ht, 5, NULL, NULL); + HCHK(freed == 0, "sweep before expiry freed %u\n", freed); + freed = pcache_ht_sweep(ht, 20, NULL, NULL); + HCHK(freed == 100, "sweep freed %u of 100\n", freed); + HCHK(ht->ovf_count == 0, "sweep left %u in overflow\n", + ht->ovf_count); + + k.s = "stay"; k.len = 4; + rc = pcache_ht_fetch(ht, &k, &out); + HCHK(rc == 0 && out.len == 4, "never-expiring key swept\n"); + pkg_free(out.s); + HCHK(pcache_ht_remove(ht, &k) == 1, "stay remove\n"); + for (i = 0, nb_used = 0; i < ht->nbuckets; i++) + nb_used += bkt_used(bucket_at(ht, i)); + HCHK(nb_used == 0, "%u slots used after the sweep test\n", + nb_used); + } + + /* CP-06 counter sanity: every create matched by a destroy after the + * full drain, and a single process never retries against itself */ + { + pcache_ht_totals_t t; + + pcache_ht_totals(ht, &t); + HCHK(t.hits > 0 && t.misses > 0 && t.stores > 0 && t.removes > 0, + "dead counters: h=%lu m=%lu s=%lu r=%lu\n", + t.hits, t.misses, t.stores, t.removes); + HCHK(t.created == t.destroyed, + "record leak: created %lu, destroyed %lu\n", + t.created, t.destroyed); + HCHK(t.entries == 0, "%lu entries after the drain\n", t.entries); + HCHK(t.retries == 0 && t.fallbacks == 0, + "single-process retries %lu, fallbacks %lu\n", + t.retries, t.fallbacks); + } + + /* CP-09 growth: fill a small table past its load factor, split it + * down, and prove every key survives the relink + re-routing */ + { + pcache_htable_t *g = pcache_htable_new(4); /* 16 buckets */ + unsigned int nb0, grown, miss = 0; + HCHK(g != NULL, "growth table creation failed\n"); + for (i = 0; i < 1000; i++) { + k.len = snprintf(kb, sizeof kb, "grow-%04u", i); k.s = kb; + v.len = snprintf(vb, sizeof vb, "gv-%04u", i); v.s = vb; + HCHK(pcache_ht_store(g, &k, &v, 0) == 0, "grow store %u\n", i); + } + nb0 = g->nbuckets; + HCHK(nb0 == 16, "unexpected initial buckets %u\n", nb0); + grown = pcache_ht_grow(g, 2, 100000); /* target LF 2 */ + HCHK(g->nbuckets > nb0, "table did not grow (%u)\n", g->nbuckets); + HCHK(g->nbuckets * 2 >= 1000, "grew short: %u buckets for 1000 " + "entries at LF 2\n", g->nbuckets); + /* every key still findable after the splits */ + for (i = 0; i < 1000; i++) { + k.len = snprintf(kb, sizeof kb, "grow-%04u", i); k.s = kb; + v.len = snprintf(vb, sizeof vb, "gv-%04u", i); + if (pcache_ht_fetch(g, &k, &out) != 0) { miss++; continue; } + if (out.len != (unsigned int)v.len || memcmp(out.s, vb, v.len)) + miss++; + pkg_free(out.s); + } + HCHK(miss == 0, "%u of 1000 keys lost/wrong after growth " + "(%u->%u buckets, %u splits)\n", miss, nb0, g->nbuckets, grown); + LM_INFO("htable selftest: growth %u->%u buckets (%u splits), " + "all 1000 keys intact\n", nb0, g->nbuckets, grown); + } + + /* every bucket must end on an even (stable) version */ + for (i = 0; i < ht->nbuckets; i++) + HCHK(!(bucket_at(ht, i)->version & 1), + "bucket %u left with an odd version\n", i); + + LM_NOTICE("htable selftest: PASS (16 buckets, overflow exercised, " + "versionless bump verified)\n"); + return 0; +} diff --git a/modules/cachedb_perf/pcache_htable.h b/modules/cachedb_perf/pcache_htable.h new file mode 100644 index 00000000000..427e1f641bb --- /dev/null +++ b/modules/cachedb_perf/pcache_htable.h @@ -0,0 +1,319 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _PCACHE_HTABLE_H_ +#define _PCACHE_HTABLE_H_ + +#include +#include + +#include "../../str.h" +#include "../../locking.h" + +#define PCACHE_SLOTS 6 +#define PCACHE_SEG_BITS 12 +#define PCACHE_SEG_SIZE (1U << PCACHE_SEG_BITS) /* 4096 buckets */ +#define PCACHE_NSEGS (1U << (24 - PCACHE_SEG_BITS)) /* for 2^24 max */ +#define PCACHE_SEQ_RETRIES 64 +#define PCACHE_OVF_BUCKETS 1024 +/* per-process stat shards: sized to a fixed cap, not counted_max_processes + * (not yet final when the table is built in mod_init, pre-fork). The + * owner:12 bucket field already caps the system at 4096 processes. */ +#define PCACHE_MAX_PROCS 1024 + +/* + * The record (DESIGN 3.3). Byte 0 is the arena class id, stamped by the + * arena and read-only here (pcache_arena.h). vlen and expires are + * naturally aligned so their loads/stores are single-copy-atomic: expires + * is the versionless-TTL-bump target (DESIGN 2.7) and vlen may be read by + * an optimistic reader mid-update. While a cell sits on a free list the + * link overlays bytes 8-15 (expires/hash) - never klen/vlen, so even a + * freed cell keeps a bounded length at the vlen offset. + */ +typedef struct pcache_rec { + unsigned char cls; /* arena class - read-only */ + unsigned char rflags; /* PCACHE_F_INT etc. (CP-04) */ + unsigned short klen; + unsigned int vlen; + volatile unsigned int expires; /* absolute ticks, 0 = never */ + unsigned int hash; /* full hash: split relink + fast reject */ + char data[]; /* key, then value, contiguous */ +} pcache_rec_t; + +#define PCACHE_REC_HDR 16 +#define PCACHE_REC_SIZE(_kl, _vl) (PCACHE_REC_HDR + (_kl) + (_vl)) + +/* rflags: native int64 counter (CP-04) - the value payload is 8 raw + * bytes, arithmetic is fixed-width under the bucket lock, and every + * user-facing read (fetch, walker) formats it as a decimal string */ +#define PCACHE_F_INT 0x01 + +/* strict bounded decimal parse; no overflow guard - counter territory */ +static inline int pcache_str2ll(const char *p, int len, long long *out) +{ + long long v = 0; + int i = 0, neg = 0; + + if (len <= 0) + return -1; + if (p[0] == '-' || p[0] == '+') { + neg = p[0] == '-'; + if (++i == len) + return -1; + } + for (; i < len; i++) { + if (p[i] < '0' || p[i] > '9') + return -1; + v = v * 10 + (p[i] - '0'); + } + *out = neg ? -v : v; + return 0; +} + +_Static_assert(offsetof(pcache_rec_t, vlen) == 4 && + offsetof(pcache_rec_t, expires) == 8 && + offsetof(pcache_rec_t, data) == PCACHE_REC_HDR, + "pcache_rec field alignment broken"); + +/* + * The bucket (DESIGN 3.1): exactly one cache line. meta packs + * used:4 (low bits) | owner:12 (process_no+1 of the lock holder, 0 = + * none) - the owner exists so the maintenance worker can detect a dead + * holder (3.5b). tags[] plus meta form one aligned 8-byte word at offset + * 8, which the SWAR tag scan loads whole. + */ +typedef struct pcache_bucket { + volatile unsigned int version; /* seqlock: odd = writer inside */ + gen_lock_t lock; /* writers (+ reader fallback) */ + unsigned char tags[PCACHE_SLOTS]; /* hash>>24, never 0 */ + volatile unsigned short meta; /* used:4 | owner:12 */ + pcache_rec_t *slot[PCACHE_SLOTS]; +} __attribute__((aligned(64))) pcache_bucket_t; + +_Static_assert(sizeof(pcache_bucket_t) == 64, + "cachedb_perf requires a 4-byte lock backend (futex/fastlock): " + "gen_lock_t made pcache_bucket exceed one cache line"); +_Static_assert(offsetof(pcache_bucket_t, tags) == 8, + "tags+meta must form the aligned 8-byte word at offset 8"); + +struct povf; + +/* + * Per-process op counters (CP-06): one cache line per process per table, + * plain increments on the owner's own line, summed only at read time. + * NEVER update_stat() per operation - that is one shared atomic line, + * the measured 0.72x collapse (DESIGN 2.5) installed by observability. + */ +typedef struct pcache_pstat { + unsigned long hits, misses, stores, removes, + created, destroyed, expired, retries, fallbacks; +} __attribute__((aligned(64))) pcache_pstat_t; + +typedef struct pcache_ht_totals { + unsigned long hits, misses, stores, removes, + created, destroyed, expired, retries, fallbacks, entries; +} pcache_ht_totals_t; + +typedef struct pcache_htable { + /* the 3.4 routing word: (level << 32) | split, published whole. + * On its own line - everything else here mutates */ + /* (level << 32) | split - genuinely 64 bits, so NOT unsigned long: + * that is 32 bits on every ILP32 target (arm32, i386) and the packing + * would collapse silently. */ + volatile uint64_t route; + char _pad0[56]; + + unsigned int nbuckets; + volatile unsigned int ovf_count; /* readers' overflow gate */ + gen_lock_t ovf_lock; + struct povf **ovf_tab; /* PCACHE_OVF_BUCKETS heads */ + + pcache_bucket_t *seg[PCACHE_NSEGS]; + + /* per-bucket min-expires hints (CP-05), parallel to seg[]: the 64B + * bucket is full, and a separate array sweeps better anyway - 16 + * hints per cache line, no bucket touched unless due. Written under + * the bucket lock, only when a LOWER expiry arrives (a TTL bump only + * raises, so the hot bump path never writes here); a stale-low hint + * just costs one wasted bucket visit. 0 = nothing expiring */ + unsigned int *hint_seg[PCACHE_NSEGS]; + + /* CP-06 counters, indexed by process_no */ + pcache_pstat_t *pstats; + unsigned int pstats_n; + + /* Baseline for perf_stats_reset: the shard sums as of the last reset. + * The counters themselves are never rewound - the hot paths own their + * own cache lines and must not be written from another process - so a + * reset just records where to count from, and pcache_ht_totals() + * reports the difference. 'entries' is a live gauge computed from the + * raw created/destroyed, so it survives a reset untouched. */ + pcache_ht_totals_t base; +} pcache_htable_t; + +/* sum the per-process shards, less the reset baseline; entries is absolute */ +void pcache_ht_totals(pcache_htable_t *ht, pcache_ht_totals_t *out); + +/* re-baseline the cumulative counters: everything perf_stats reports as a + * running total starts from zero again. Live gauges (entries, buckets, + * overflow, arena) are unaffected. */ +void pcache_ht_stats_reset(pcache_htable_t *ht); + +/* current live bucket count (grows at runtime, CP-09) - for the CP-11 + * growth event, which reports the before/after span */ +unsigned int pcache_ht_nbuckets(pcache_htable_t *ht); + +pcache_htable_t *pcache_htable_new(unsigned int size_log2); + +/* 0 = stored; -1 = error; -2 = out of memory (the arena could not allocate + * a cell - the cache is full and the write was dropped). @expires is + * absolute ticks, 0 = never */ +int pcache_ht_store(pcache_htable_t *ht, const str *key, const str *val, + unsigned int expires); + +/* get_buf(): @buf was too small. *vlen stays 0 and *needed carries the + * size the value would have needed - never a length the caller could + * mistake for "bytes written into buf". */ +#define PCACHE_E_TOOSMALL (-3) + +/* Smallest buffer get_buf() will accept. A native counter is 8 raw bytes + * formatted as decimal on read, so anything shorter could not represent + * every legal hit; enforced rather than assumed. */ +#define PCACHE_GETBUF_MIN 24 + +/* + * Allocation-free read into a caller-owned buffer: the value is copied + * once, straight from the record to @buf, instead of being copied to the + * internal scratch and then into a freshly pkg_malloc'd str. + * + * @buf MUST be private to the calling process (its own stack or pkg). The + * lock-free read path writes into it SPECULATIVELY - a retried optimistic + * section may leave a partial value behind - so on any return other than 0 + * the contents are undefined and must not be used. The value is not + * NUL-terminated. + * + * 0 = hit, *vlen bytes written (always <= @buflen); -2 = miss or expired; + * -1 = error or malformed request; PCACHE_E_TOOSMALL = value does not fit, + * *needed holds the required size. *vlen and *needed are zeroed first. + * @needed may be NULL. + */ +/* Existence probe: is @key present and live, how long is its value and + * when does it expire - without copying the value anywhere. Shares the + * whole read path with the fetches, stopping before the copy-out, so it + * can never disagree with a read about whether a key is there. + * + * Cheaper than any fetch by construction: no pkg_malloc (the ~70 ns the + * profile attributes to the allocator), no copy, and the record's payload + * is never touched - a miss is usually settled on the bucket's tag word + * alone. Intended for answering "do I have this key?" - a cross-node + * lookup asking peers, or a script/MI existence test. + * + * @vlen, @expires and @is_counter are optional; @expires is absolute ticks + * (0 = never); @is_counter reports a native counter, whose value is a + * per-node quantity rather than a portable one. + * @return 0 = present and live, -2 = absent or expired, -1 = bad args. */ +int pcache_ht_probe(pcache_htable_t *ht, const str *key, unsigned int *vlen, + unsigned int *expires, int *is_counter); + +int pcache_ht_fetch_buf(pcache_htable_t *ht, const str *key, char *buf, + unsigned int buflen, unsigned int *vlen, unsigned int *needed); + +/* 0 = hit (val->s pkg-allocated, caller frees); -2 = miss or expired; + * -1 = error */ +int pcache_ht_fetch(pcache_htable_t *ht, const str *key, str *val); + +/* as pcache_ht_fetch, plus *@expires = the record's absolute expiry (0 = + * never) on a hit - the MI perf_get reports the TTL with the value */ +int pcache_ht_fetch_ex(pcache_htable_t *ht, const str *key, str *val, + unsigned int *expires); + +/* 1 = removed; 0 = was absent; -1 = error */ +int pcache_ht_remove(pcache_htable_t *ht, const str *key); + +/* re-arm an existing key's TTL without rewriting the value (MI perf_ttl): + * one aligned store of expires under the bucket lock, the versionless bump + * of 2.7. @expires is absolute ticks (0 = never). 1 = re-armed, 0 = absent */ +int pcache_ht_touch(pcache_htable_t *ht, const str *key, unsigned int expires); + +/* atomic counter add (CP-04): creates a native counter on an absent key, + * accumulates fixed-width on an existing one, converts a numeric string + * record on first touch. 0 = ok (*new_val = the result); -1 = error or + * the existing value is not an integer. @expires re-arms the TTL, + * absolute ticks, 0 = never */ +int pcache_ht_add(pcache_htable_t *ht, const str *key, long long delta, + unsigned int expires, long long *new_val); + +/* + * Key/value walker: per-slot optimistic snapshots over every bucket, then + * the overflow chains under the overflow lock. @key/@val given to the + * callback are stable NUL-terminated copies in walker-owned buffers, + * valid only for the duration of the call; @expires is raw (0 = never) - + * filtering is the callback's choice. Return <0 from the callback to + * stop the walk (returned through). + * + * Guarantees are the Redis SCAN class: an entry mutated concurrently may + * be seen once, twice or not at all. The overflow leg runs under the + * overflow lock, so the callback must not re-enter this cache. + */ +typedef int (*pcache_iter_cb)(const str *key, const str *val, + unsigned int expires, void *ctx); +int pcache_ht_iter(pcache_htable_t *ht, pcache_iter_cb cb, void *ctx); + +/* default buckets visited per perf_scan call when count is unset */ +#define PCACHE_SCAN_BUCKETS 100 + +/* + * Cursored, bounded walk (MI perf_scan, Redis SCAN semantics). Starts at + * bucket *@cursor, visits up to @max_buckets buckets calling @cb per live + * entry, and updates *@cursor to the bucket to resume from - 0 once the walk + * (overflow leg included) is complete. Pass *@cursor = 0 to begin. The + * ascending cursor is stable across a concurrent resize (3.4 / 5.2) and gives + * the >=-once guarantee; it advances a whole bucket at a time. 0, or <0 on + * error / callback stop. + */ +int pcache_ht_scan(pcache_htable_t *ht, unsigned int *cursor, + unsigned int max_buckets, pcache_iter_cb cb, void *ctx); + +/* CP-11: invoked for each record the sweep reaps, after the bucket lock is + * released and while the key is still valid, so the caller can raise an + * expiry event. @key points into the about-to-be-freed record; do not + * retain it past the call. */ +typedef void (*pcache_expired_cb)(const str *key, void *ctx); + +/* expiry sweep (CP-05): visits only buckets whose hint is due, removes + * expired records (overflow chains too whenever any overflow exists) and + * reclaims their cells through the global pool - the sweeping process is + * not an allocator, so private-stack frees would never drain. If @cb is + * non-NULL it is called once per reaped record (CP-11). Returns the number + * of records reclaimed. */ +unsigned int pcache_ht_sweep(pcache_htable_t *ht, unsigned int now, + pcache_expired_cb cb, void *cb_ctx); + +/* linear-hash growth (CP-09): split buckets while entries/nbuckets exceeds + * @target_lf, up to @budget splits. Single-splitter (maintenance timer). */ +unsigned int pcache_ht_grow(pcache_htable_t *ht, unsigned int target_lf, + unsigned int budget); + +/* modparam-triggered startup selftest; -1 on any mismatch */ +int pcache_htable_selftest(void); + +#endif /* _PCACHE_HTABLE_H_ */ diff --git a/modules/cachedb_perf/pcache_mem.c b/modules/cachedb_perf/pcache_mem.c new file mode 100644 index 00000000000..b82b2a60fc2 --- /dev/null +++ b/modules/cachedb_perf/pcache_mem.c @@ -0,0 +1,312 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/* + * Huge-page tier detection (DESIGN 2.6.1 / CP-20, detection half). + * + * Every tier is detected by TRYING it on a scratch mapping and verifying + * the result through /proc/self/smaps - never inferred from the kernel + * version or from sysfs configuration (the 6.8/6.12 MADV_COLLAPSE + * divergence proves such checks lie). The scratch mapping is unmapped + * after the probe; the never-unmap invariant (DESIGN 3.2) applies to the + * arena, which holds entries - not to a probe that never does. + * + * The probe is advisory: the arena allocator (CP-02/CP-20) re-runs the + * ladder per chunk, so a pool that appears or drains after startup is + * handled at allocation time. This runs pre-fork, from mod_init. + */ + +#include +#include +#include +#include +#include +#include + +#include "../../dprint.h" + +#include "pcache_mem.h" + +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0x40000 +#endif +#ifndef MADV_HUGEPAGE +#define MADV_HUGEPAGE 14 +#endif +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif + +#define PCACHE_HPS (2UL * 1024 * 1024) /* huge page size, x86_64 */ + +struct pcache_mem_info pcache_mem; + +static int read_vm_int(const char *path) +{ + FILE *f; + int v = -1; + + f = fopen(path, "r"); + if (!f) + return -1; + if (fscanf(f, "%d", &v) != 1) + v = -1; + fclose(f); + return v; +} + +/* global huge shmem, /proc/meminfo "ShmemHugePages:" in kB. This is the + * verification for MADV_COLLAPSE: a shmem collapse creates the huge folio + * but does NOT install the PMD mapping in the caller's page table, so + * per-process smaps shows nothing until a re-fault - the bench verified + * through this same global counter (DESIGN 2.6.1) */ +static long read_shmem_huge_kb(void) +{ + FILE *f; + char line[256]; + long kb = -1; + + f = fopen("/proc/meminfo", "r"); + if (!f) + return -1; + while (fgets(line, sizeof line, f)) { + if (!strncmp(line, "ShmemHugePages:", 15)) { + kb = strtol(line + 15, NULL, 10); + break; + } + } + fclose(f); + return kb; +} + +/* is the 2M range starting at @addr PMD-mapped in this process? + * ("verify, never infer" - DESIGN 2.6.1) */ +static int range_is_huge(unsigned long addr) +{ + FILE *f; + char line[256], *p; + unsigned long start, end, kb; + int in_range = 0, huge = 0; + + f = fopen("/proc/self/smaps", "r"); + if (!f) + return 0; + + while (fgets(line, sizeof line, f)) { + if (sscanf(line, "%lx-%lx ", &start, &end) == 2) { + in_range = (start <= addr && addr < end); + continue; + } + if (!in_range) + continue; + if (!strncmp(line, "AnonHugePages:", 14) || + !strncmp(line, "ShmemPmdMapped:", 15) || + !strncmp(line, "FilePmdMapped:", 14)) { + p = strchr(line, ':'); + kb = strtoul(p + 1, NULL, 10); + if (kb >= PCACHE_HPS / 1024) { + huge = 1; + break; + } + } + } + + fclose(f); + return huge; +} + +void pcache_mem_probe(void) +{ + char *resv, *aligned; + void *p; + size_t len; + long shmem_kb; + int rc; + + memset(&pcache_mem, 0, sizeof pcache_mem); + pcache_mem.tier = PCACHE_MEM_4K; + + pcache_mem.huge_static = + read_vm_int("/proc/sys/vm/nr_hugepages"); + pcache_mem.huge_overcommit = + read_vm_int("/proc/sys/vm/nr_overcommit_hugepages"); + + /* tier 1: MAP_HUGETLB. Pages are secured against the pool (static + * or overcommit) at mmap time, so a successful map plus one touched + * byte proves the route; failure (ENOMEM/EINVAL) drops a tier */ + p = mmap(NULL, PCACHE_HPS, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + *(volatile char *)p = 1; + munmap(p, PCACHE_HPS); + pcache_mem.tier = PCACHE_MEM_HUGETLB; + return; + } + + /* Tiers 2 and 3 need a 2M-aligned shmem scratch, and aligning the VA + * inside an unaligned mapping is NOT enough: shmem THP requires the + * VA and the shmem *file offset* to be congruent mod 2M, and offset + * 0 is pinned to wherever the mapping starts. A VA-aligned range + * inside an unaligned mapping sits at offset != 0 there and is + * simply ineligible (THPeligible 0, MADV_COLLAPSE EINVAL) - found + * the hard way: the probe passed standalone and failed in-process + * purely on ASLR luck. So: reserve VA PROT_NONE first, then + * MAP_FIXED the shmem at a 2M boundary inside the reservation - an + * atomic replace, no race with other mappings. */ + len = 2 * PCACHE_HPS; + resv = mmap(NULL, len, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (resv == MAP_FAILED) + return; + aligned = (char *)(((unsigned long)resv + PCACHE_HPS - 1) + & ~(PCACHE_HPS - 1)); + p = mmap(aligned, PCACHE_HPS, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + munmap(resv, len); + return; + } + + /* tier 2: advice set before first touch -> huge at fault time */ + rc = madvise(aligned, PCACHE_HPS, MADV_HUGEPAGE); + memset(aligned, 1, PCACHE_HPS); + if (rc == 0 && range_is_huge((unsigned long)aligned)) { + pcache_mem.tier = PCACHE_MEM_THP_ADVISE; + goto out; + } + + /* tier 3: collapse the already-faulted 4K pages in place. Verified + * by the ShmemHugePages delta, not smaps: shmem collapse creates the + * huge folio without PMD-mapping it here (later faults do that) */ + shmem_kb = read_shmem_huge_kb(); + rc = madvise(aligned, PCACHE_HPS, MADV_COLLAPSE); + if (rc == 0 && shmem_kb >= 0 && + read_shmem_huge_kb() - shmem_kb >= (long)(PCACHE_HPS / 1024)) { + pcache_mem.tier = PCACHE_MEM_THP_COLLAPSE; + goto out; + } + if (rc != 0) + LM_DBG("MADV_COLLAPSE: %s\n", strerror(errno)); + +out: + munmap(resv, len); +} + +/* + * CP-20: reserve a large 2M-aligned MAP_SHARED region for the arena, backed + * by huge pages via the same ladder as the probe, mlock-pinned against swap. + * Created pre-fork and never unmapped, so every worker inherits it (the + * invariant the lock-free read path and CP-09 growth both need). Returns + * the base (NULL on total failure -> caller falls back to shm_malloc), + * sets *tier to what was achieved and *locked_mb to the pinned amount. + */ +void *pcache_mem_reserve(size_t size, enum pcache_mem_tier *tier, + unsigned long *locked_mb) +{ + size_t asize = (size + PCACHE_HPS - 1) & ~(PCACHE_HPS - 1); + char *resv, *base; + long shmem_kb; + void *p; + + *locked_mb = 0; + *tier = PCACHE_MEM_4K; + + /* tier 1: MAP_HUGETLB - unswappable, exempt from RLIMIT_MEMLOCK */ + p = mmap(NULL, asize, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + memset(p, 0, asize); /* commit the pool pages */ + *tier = PCACHE_MEM_HUGETLB; + /* + * No mlock() call needed here (tier 1 is already unswappable by + * construction, per the comment above) - but report the nominal + * size as pinned anyway, matching HG_MALLOC's own hg_mem_reserve() + * convention for the identical tier-1 case. Leaving this at the + * init'd 0 was technically true (no mlock() syscall happened) but + * reads, side by side with HG_MALLOC's own tier-1 NOTICE line, as + * "this reservation is unprotected against swap" - which is false; + * it is exactly as protected as HG_MALLOC's, just via a different + * mechanism. Caught live during a real diagnosis session (2026-08-07) + * by the same kind of confusion the tier_probe/tier_active split + * above was written to eliminate. + */ + *locked_mb = asize >> 20; + return p; + } + + /* tiers 2-4: 2M-aligned MAP_SHARED|ANON (reserve PROT_NONE, then + * MAP_FIXED at a 2M boundary - VA/offset congruence, DESIGN 2.6.1) */ + resv = mmap(NULL, asize + PCACHE_HPS, PROT_NONE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (resv == MAP_FAILED) + return NULL; + base = (char *)(((unsigned long)resv + PCACHE_HPS - 1) + & ~(PCACHE_HPS - 1)); + p = mmap(base, asize, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + munmap(resv, asize + PCACHE_HPS); + return NULL; + } + + /* advise huge before first touch (tier 2), then pin+populate: a cold + * mlock populates to pin, so it doubles as the pre-fault (DESIGN 2.6.2) */ + madvise(base, asize, MADV_HUGEPAGE); + shmem_kb = read_shmem_huge_kb(); + if (mlock(base, asize) == 0) { + *locked_mb = asize >> 20; + } else { + LM_WARN("mlock of the %zu MB arena failed (%s): continuing " + "unpinned (swappable). If running under systemd, add " + "LimitMEMLOCK=infinity to the unit.\n", + asize >> 20, strerror(errno)); + memset(base, 0, asize); /* still pre-fault */ + } + + if (range_is_huge((unsigned long)base)) { + *tier = PCACHE_MEM_THP_ADVISE; + } else if (shmem_kb >= 0 && + madvise(base, asize, MADV_COLLAPSE) == 0 && + read_shmem_huge_kb() - shmem_kb >= (long)(asize / 1024)) { + *tier = PCACHE_MEM_THP_COLLAPSE; + } else { + *tier = PCACHE_MEM_4K; /* reserved+pinned but 4K */ + } + return base; +} + +const char *pcache_mem_tier_str(enum pcache_mem_tier tier) +{ + switch (tier) { + case PCACHE_MEM_HUGETLB: + return "MAP_HUGETLB 2M pages"; + case PCACHE_MEM_THP_ADVISE: + return "THP 2M pages via MADV_HUGEPAGE (huge at fault)"; + case PCACHE_MEM_THP_COLLAPSE: + return "THP 2M pages via MADV_COLLAPSE (post-fill retrofit)"; + case PCACHE_MEM_4K: + return "plain 4K pages"; + case PCACHE_MEM_NO_ARENA: + return "core shm_malloc - no dedicated arena; page backing follows " + "the core allocator (2M hugepages under HG_MALLOC)"; + } + return "unknown"; +} diff --git a/modules/cachedb_perf/pcache_mem.h b/modules/cachedb_perf/pcache_mem.h new file mode 100644 index 00000000000..29c76cdecb2 --- /dev/null +++ b/modules/cachedb_perf/pcache_mem.h @@ -0,0 +1,58 @@ +/* + * cachedb_perf - high-performance local memory cache + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef _PCACHE_MEM_H_ +#define _PCACHE_MEM_H_ + +/* the four-tier huge-page ladder (DESIGN 2.6.1 / CP-20), best first */ +enum pcache_mem_tier { + PCACHE_MEM_HUGETLB = 1, /* mmap MAP_HUGETLB - best, 1.42x on chases */ + PCACHE_MEM_THP_ADVISE, /* shmem THP via MADV_HUGEPAGE, huge at fault */ + PCACHE_MEM_THP_COLLAPSE, /* shmem THP via MADV_COLLAPSE, post-fill */ + PCACHE_MEM_4K, /* plain pages - always works */ + /* Not a backing tier at all: there is no dedicated reservation, so + * every allocation goes through the core's shm_malloc() and the real + * page backing is whatever the CORE allocator uses (under HG_MALLOC + * that is 2M hugepages). Reporting 4K here was wrong - it named a + * property of an arena that does not exist and read as "your cache + * is on small pages" when it may well not be. */ + PCACHE_MEM_NO_ARENA = 99, +}; + +struct pcache_mem_info { + enum pcache_mem_tier tier; + int huge_static; /* vm.nr_hugepages at probe time, -1 unknown */ + int huge_overcommit; /* vm.nr_overcommit_hugepages, -1 unknown */ +}; + +extern struct pcache_mem_info pcache_mem; + +/* probe the ladder by trying each route on a scratch mapping; pre-fork only */ +void pcache_mem_probe(void); +const char *pcache_mem_tier_str(enum pcache_mem_tier tier); + +/* CP-20: reserve a huge-page-backed, mlock-pinned, 2M-aligned MAP_SHARED + * region for the arena (pre-fork, never unmapped). NULL -> use shm_malloc. */ +void *pcache_mem_reserve(size_t size, enum pcache_mem_tier *tier, + unsigned long *locked_mb); + +#endif /* _PCACHE_MEM_H_ */ diff --git a/modules/cachedb_perf/pull_api.h b/modules/cachedb_perf/pull_api.h new file mode 100644 index 00000000000..ead547d3c19 --- /dev/null +++ b/modules/cachedb_perf/pull_api.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2026 VoIPcloud + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * Asynchronous cross-node pull, for a consumer that would rather suspend + * a transaction than occupy a process while the cluster answers. + * + * pcache_pull_api_t pull; + * if (load_pcache_pull_api(&pull) == 0) ... (mod_init) + * + * rc = pull.start(con, &key, &fd, &handle); + * if (rc == 1) -> wait on @fd, then call finish() + * if (rc == 0) -> the cluster has already answered: nobody has it + * if (rc < 0) -> cannot pull (not enabled, no peers, no free slot) + * + * Binding is optional by design: a consumer that cannot find this simply + * does not do cross-node lookups, exactly as today. + */ + +#ifndef PCACHE_PULL_API_H +#define PCACHE_PULL_API_H + +#include "../../str.h" +#include "../../sr_module.h" +#include "../../cachedb/cachedb.h" + +/* Begin a pull for @key on @con's collection. On success @fd becomes + * readable once an answer has landed (or the request has been settled), + * and @handle identifies it to finish(). + * @return 1 = started, 0 = already known absent, -1 = cannot pull. */ +typedef int (*pcache_pull_start_f)(cachedb_con *con, str *key, int *fd, + unsigned int *handle); + +/* As start(), but ask one node first instead of the whole cluster. The + * caller supplies @node_id from whatever knowledge it has of where the + * key was put; it is treated as a hint and validated against current + * membership, so a stale or nonsensical one costs nothing but the usual + * broadcast. @node_id <= 0 behaves exactly like start(). */ +typedef int (*pcache_pull_start_at_f)(cachedb_con *con, str *key, + int node_id, int *fd, unsigned int *handle); + +/* This node's id in the cluster the cache is part of, 0 if it has none. + * A consumer that wants to record where it stored something needs this, + * and getting it from here saves it from binding the clusterer itself. */ +typedef int (*pcache_my_node_id_f)(cachedb_con *con); + +/* Collect a started pull. Safe to call after a timeout as well as after + * the descriptor fires - it releases the request either way, so a caller + * that gives up leaks nothing. On a hit @val is pkg memory the caller + * owns; the value has also been stored locally, so an ordinary get will + * now find it. + * @return 1 = value in @val, 0 = definitively absent, -1 = no answer. */ +typedef int (*pcache_pull_finish_f)(cachedb_con *con, str *key, + unsigned int handle, str *val); + +typedef struct pcache_pull_api { + pcache_pull_start_f start; + pcache_pull_finish_f finish; + pcache_pull_start_at_f start_at; + pcache_my_node_id_f my_node_id; +} pcache_pull_api_t; + +typedef int (*load_pcache_pull_f)(pcache_pull_api_t *api); + +static inline int load_pcache_pull_api(pcache_pull_api_t *api) +{ + load_pcache_pull_f load_it; + + load_it = (load_pcache_pull_f)(void *)find_export("load_pcache_pull", 0); + if (!load_it) + return -1; + return load_it(api); +} + +#endif /* PCACHE_PULL_API_H */