diff --git a/Makefile.conf.template b/Makefile.conf.template index c6396cc5f49..8f98b693754 100644 --- a/Makefile.conf.template +++ b/Makefile.conf.template @@ -18,6 +18,7 @@ #db_oracle= Provides Oracle connectivity for OpenSIPS. | Development library of OCI, typically instantclient-sdk-10.2.0.3 #db_perlvdb= Provides a virtualization framework for OpenSIPS's database access. | Perl library development files, typically libperl-dev #db_postgres= Provides Postgres connectivity for OpenSIPS | PostgreSQL library and development library - typically libpq5 and libpq-dev +#db_redis= Provides Redis (single instance or cluster) row storage for OpenSIPS | Redis client library, hiredis #db_sqlite= Provides SQLite connectivity for OpenSIPS | SQLite library and development library - typically libsqlite3 and libsqlite3-dev #db_unixodbc= Allows to use the unixodbc package with OpenSIPS | ODBC library and ODBC development library #dialplan= Implements generic string translations based on matching and replacement rules | PCRE development library, typically libpcre2-dev @@ -76,7 +77,7 @@ #uuid= UUID generator | uuid-dev # Modules omitted from the default build, generally due to external dependencies. -exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_sqlite db_unixodbc dialplan emergency event_rabbitmq event_kafka event_sqs h350 httpd http2d identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dialoginfo presence_mwi presence_reginfo presence_xml presence_dfks proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python regex rabbitmq_consumer rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp +exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates clusterer_controller compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_redis db_sqlite db_unixodbc dialplan emergency event_kafka event_rabbitmq event_sqs h350 http2d httpd identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dfks presence_dialoginfo presence_mwi presence_reginfo presence_xml proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python rabbitmq_consumer regex rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp # Modules forced into the build, even when also listed in exclude_modules. include_modules?= @@ -97,6 +98,17 @@ DEFS+= -DQ_MALLOC #Quality assurance memory allocator with runtime safety checks DEFS+= -DHP_MALLOC #High performance allocator with fine-grained locking DEFS+= -DDBG_MALLOC #Include additional, debug-enabled allocator flavors DEFS+= -DF_PARALLEL_MALLOC #Fast memory allocator with paralel buckets +DEFS+= -DHG_MALLOC #Hugepage-backed slab allocator with a lock-free fast path +# HG_MALLOC reserves its arena up front on 2 MB huge pages, pre-faults and +# mlock()s it, and serves fixed size classes from per-thread caches so the +# common path takes no lock. A block belongs to one size class while in use; +# once it drains completely it returns to the class-agnostic buddy and can be +# re-carved for a different class, so the carve shrinks again after a burst. +# Because the arena is pinned, resident memory equals -m plus -M x processes +# from the first second - unlike the mmap-backed allocators, oversizing is not +# cheap. It needs a huge-page pool of at least (-m + -M x procs) / 2 pages and +# a raised memlock limit; without them it drops to a lower tier and keeps +# running. See mem/README.hg_malloc and mem/README.hg_arena_v2 #DEFS+= -DQM_DBG_MALLOC_HIST=3 #Size of debug history for Q_MALLOC (default is 1) #DEFS+= -DNO_DEBUG #Compile out all debug messages #DEFS+= -DNO_LOG #Compile out all logging diff --git a/Makefile.defs b/Makefile.defs index ab003c6a893..5bc5dcbc8d2 100644 --- a/Makefile.defs +++ b/Makefile.defs @@ -524,10 +524,12 @@ endif ifeq (,$(findstring F_MALLOC, $(DEFS))) ifeq (,$(findstring Q_MALLOC, $(DEFS))) ifeq (,$(findstring HP_MALLOC, $(DEFS))) +ifeq (,$(findstring HG_MALLOC, $(DEFS))) DEFS+= -DQ_MALLOC endif endif endif +endif ifneq (,$(MEM_STATS_HDR)) EXPECTED_MD5=$(shell [ -f $(MEM_STATS_HDR) ] && tail -n 1 $(MEM_STATS_HDR) | awk '{print $$4}') @@ -558,6 +560,10 @@ MKTAGS=ctags -R . # decently fast allocator with extra sanity checks + safety buffers # -DHP_MALLOC # high performance allocator optimized for shared memory multiprocessing +# -DHG_MALLOC +# hugepage-backed slab allocator; fixed size classes bump-carved from +# its own huge-page (or best-effort degraded) reservation, no fallback +# to another allocator on exhaustion - increase -m/-M instead # -DDBG_MALLOC # will cause each of the enabled allocators to also include a "debugging" # flavor of itself; by running a debugging version of an allocator, you diff --git a/cfg.lex b/cfg.lex index bd89db1f46a..43378d5de1a 100644 --- a/cfg.lex +++ b/cfg.lex @@ -246,6 +246,11 @@ RPM_MEM_FILE "restart_persistency_cache_file" RPM_MEM_SIZE "restart_persistency_size" MEMLOG "memlog"|"mem_log" MEMDUMP "memdump"|"mem_dump" +PIN_WORKERS "pin_workers"|"cpu_pinning" +PIN_UDP_CPUS "pin_udp_cpus" +PIN_TCP_CPUS "pin_tcp_cpus" +PIN_TIMER_CPUS "pin_timer_cpus" +PIN_MODULE_CPUS "pin_module_cpus" SHM_MEMLOG_SIZE "shm_memlog_size" EXECMSGTHRESHOLD "execmsgthreshold"|"exec_msg_threshold" WORKER_REACTOR_TIMEOUT "workerreactortimeout"|"worker_reactor_timeout" @@ -290,6 +295,10 @@ DB_DEFAULT_URL "db_default_url" DB_MAX_ASYNC_CONNECTIONS "db_max_async_connections" DISABLE_503_TRANSLATION "disable_503_translation" AUTO_SCALING_PROFILE "auto_scaling_profile" +SHM_AUTO_SCALING_PROFILE "shm_auto_scaling_profile" +PKG_AUTO_SCALING_PROFILE "pkg_auto_scaling_profile" +HG_RAM_FLOOR_MB "hg_ram_floor_mb" +HG_AUTOSCALE_DRY_RUN "hg_autoscale_dry_run" AUTO_SCALING_CYCLE "auto_scaling_cycle" TIMER_WORKERS "timer_workers" @@ -319,6 +328,7 @@ TICK \' SLASH "/" AS {EAT_ABLE}("as"|"AS"){EAT_ABLE} USE_WORKERS {EAT_ABLE}("use_workers"|"USE_WORKERS"){EAT_ABLE} +PIN_CPUS {EAT_ABLE}("pin_cpus"|"PIN_CPUS"){EAT_ABLE} SOCK_TOS {EAT_ABLE}("tos"|"TOS"){EAT_ABLE} USE_AUTO_SCALING_PROFILE {EAT_ABLE}("use_auto_scaling_profile"|"USE_AUTO_SCALING_PROFILE"){EAT_ABLE} SCALE_UP_TO {EAT_ABLE}("scale"|"SCALE"){EAT_ABLE}+("up"|"UP"){EAT_ABLE}+("to"|"TO"){EAT_ABLE} @@ -481,6 +491,11 @@ SPACE [ ] {RPM_MEM_SIZE} { count(); yylval.strval=yytext; return RPM_MEM_SIZE; } {MEMLOG} { count(); yylval.strval=yytext; return MEMLOG; } {MEMDUMP} { count(); yylval.strval=yytext; return MEMDUMP; } +{PIN_WORKERS} { count(); yylval.strval=yytext; return PIN_WORKERS; } +{PIN_UDP_CPUS} { count(); yylval.strval=yytext; return PIN_UDP_CPUS; } +{PIN_TCP_CPUS} { count(); yylval.strval=yytext; return PIN_TCP_CPUS; } +{PIN_TIMER_CPUS} { count(); yylval.strval=yytext; return PIN_TIMER_CPUS; } +{PIN_MODULE_CPUS} { count(); yylval.strval=yytext; return PIN_MODULE_CPUS; } {SHM_MEMLOG_SIZE} { count(); yylval.strval=yytext; return SHM_MEMLOG_SIZE; } {EXECMSGTHRESHOLD} { count(); yylval.strval=yytext; return EXECMSGTHRESHOLD; } {WORKER_REACTOR_TIMEOUT} { count(); yylval.strval=yytext; return WORKER_REACTOR_TIMEOUT; } @@ -544,6 +559,14 @@ SPACE [ ] return DISABLE_503_TRANSLATION; } {AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext; return AUTO_SCALING_PROFILE; } +{SHM_AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext; + return SHM_AUTO_SCALING_PROFILE; } +{PKG_AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext; + return PKG_AUTO_SCALING_PROFILE; } +{HG_RAM_FLOOR_MB} { count(); yylval.strval=yytext; + return HG_RAM_FLOOR_MB; } +{HG_AUTOSCALE_DRY_RUN} { count(); yylval.strval=yytext; + return HG_AUTOSCALE_DRY_RUN; } {AUTO_SCALING_CYCLE} { count(); yylval.strval=yytext; return AUTO_SCALING_CYCLE; } {TIMER_WORKERS} { count(); yylval.strval=yytext; @@ -603,6 +626,7 @@ SPACE [ ] {COMMA} { count(); return COMMA; } {SEMICOLON} { count(); return SEMICOLON; } {USE_WORKERS} { count(); return USE_WORKERS; } +{PIN_CPUS} { count(); return PIN_CPUS; } {SOCK_TOS} { count(); return SOCK_TOS; } {USE_AUTO_SCALING_PROFILE} { count(); return USE_AUTO_SCALING_PROFILE; } {COLON} { count(); return COLON; } diff --git a/cfg.y b/cfg.y index 5e6fb7fc792..2d18609ad0f 100644 --- a/cfg.y +++ b/cfg.y @@ -183,6 +183,7 @@ extern char *finame; struct listen_param { enum si_flags flags; int workers; + char *pin_cpus; int tos; struct socket_id *socket; char *tag; @@ -353,6 +354,11 @@ extern int cfg_parse_only_routes; %token RPM_MEM_SIZE %token MEMLOG %token MEMDUMP +%token PIN_WORKERS +%token PIN_UDP_CPUS +%token PIN_TCP_CPUS +%token PIN_TIMER_CPUS +%token PIN_MODULE_CPUS %token SHM_MEMLOG_SIZE %token EXECMSGTHRESHOLD %token WORKER_REACTOR_TIMEOUT @@ -407,6 +413,10 @@ extern int cfg_parse_only_routes; %token ASYNC_TOKEN %token LAUNCH_TOKEN %token AUTO_SCALING_PROFILE +%token SHM_AUTO_SCALING_PROFILE +%token PKG_AUTO_SCALING_PROFILE +%token HG_RAM_FLOOR_MB +%token HG_AUTOSCALE_DRY_RUN %token AUTO_SCALING_CYCLE %token TIMER_WORKERS @@ -461,6 +471,7 @@ extern int cfg_parse_only_routes; %token SLASH %token AS %token USE_WORKERS +%token PIN_CPUS %token SOCK_TOS %token USE_AUTO_SCALING_PROFILE %token MAX @@ -763,6 +774,9 @@ socket_def_param: ANYCAST { IFOR(); | USE_WORKERS NUMBER { IFOR(); p_tmp.workers=$2; } + | PIN_CPUS STRING { IFOR(); + p_tmp.pin_cpus=$2; + } | SOCK_TOS NUMBER { IFOR(); p_tmp.tos=$2; } @@ -1259,6 +1273,16 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR(); | SHM_MEMLOG_SIZE EQUAL error { yyerror("int value expected"); } | MEMDUMP EQUAL snumber { IFOR(); memdump=$3; } | MEMDUMP EQUAL error { yyerror("int value expected"); } + | PIN_WORKERS EQUAL snumber { IFOR(); pin_workers=$3; } + | PIN_WORKERS EQUAL error { yyerror("int value expected"); } + | PIN_UDP_CPUS EQUAL STRING { IFOR(); pin_udp_cpus=$3; pin_workers=1; } + | PIN_UDP_CPUS EQUAL error { yyerror("string value expected"); } + | PIN_TCP_CPUS EQUAL STRING { IFOR(); pin_tcp_cpus=$3; pin_workers=1; } + | PIN_TCP_CPUS EQUAL error { yyerror("string value expected"); } + | PIN_TIMER_CPUS EQUAL STRING { IFOR(); pin_timer_cpus=$3; pin_workers=1; } + | PIN_TIMER_CPUS EQUAL error { yyerror("string value expected"); } + | PIN_MODULE_CPUS EQUAL STRING { IFOR(); pin_module_cpus=$3; pin_workers=1; } + | PIN_MODULE_CPUS EQUAL error { yyerror("string value expected"); } | EXECMSGTHRESHOLD EQUAL NUMBER { IFOR();execmsgthreshold=$3; } | EXECMSGTHRESHOLD EQUAL error { yyerror("int value expected"); } | WORKER_REACTOR_TIMEOUT EQUAL NUMBER { IFOR(); worker_reactor_timeout=$3; } @@ -1644,6 +1668,26 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR(); | AUTO_SCALING_PROFILE EQUAL error { yyerror("bad auto-scaling profile definition"); } + | SHM_AUTO_SCALING_PROFILE EQUAL ID { IFOR(); + hg_shm_profile_name=$3; } + | SHM_AUTO_SCALING_PROFILE EQUAL error { + yyerror("profile name expected"); + } + | PKG_AUTO_SCALING_PROFILE EQUAL ID { IFOR(); + hg_pkg_profile_name=$3; } + | PKG_AUTO_SCALING_PROFILE EQUAL error { + yyerror("profile name expected"); + } + | HG_RAM_FLOOR_MB EQUAL NUMBER { IFOR(); + hg_ram_floor_mb=$3; } + | HG_RAM_FLOOR_MB EQUAL error { + yyerror("integer value expected"); + } + | HG_AUTOSCALE_DRY_RUN EQUAL NUMBER { IFOR(); + hg_autoscale_dry_run=$3; } + | HG_AUTOSCALE_DRY_RUN EQUAL error { + yyerror("integer value expected"); + } | AUTO_SCALING_CYCLE EQUAL NUMBER { IFOR(); auto_scaling_cycle=$3; } | AUTO_SCALING_CYCLE EQUAL error { @@ -2797,6 +2841,7 @@ static void fill_socket_id(struct listen_param *param, struct socket_id *s) while (s) { s->flags |= param->flags; s->workers = param->workers; + s->pin_cpus = param->pin_cpus; s->tos = param->tos; s->auto_scaling_profile = param->auto_scaling_profile; s->tag = param->tag; diff --git a/core_stats.c b/core_stats.c index 739a5207006..6a6f239eb01 100644 --- a/core_stats.c +++ b/core_stats.c @@ -186,6 +186,233 @@ static unsigned long get_pkg_fragments( void*proc_id) return pkg_status[(unsigned long)proc_id][PKG_FRAGMENTS_SIZE_IDX]; } +int hg_pkg_peak_all(unsigned long *peak, unsigned long *sum, int *nproc) +{ + unsigned long mx = 0, tot = 0; + int i, n = 0; + + if (!pkg_status || no_pkg_status <= 0) + return -1; + + /* + * Two passes on purpose. signal_pkg_status() only ASKS each process to + * refresh its slot - the answer arrives later over IPC - so reading in + * the same loop that asks would report whatever was there from the + * previous round, and on the very first call that is zero. Ask + * everyone, give the replies a moment to land, then read. + */ + for (i = 0; i < no_pkg_status; i++) + signal_pkg_status((unsigned long)i); + + usleep(50000); + + for (i = 0; i < no_pkg_status; i++) { + unsigned long v = pkg_status[i][PKG_MAX_USED_SIZE_IDX]; + + /* a slot with no total_size has never been filled in: that + * process either does not exist or has not answered yet */ + if (!pkg_status[i][PKG_TOTAL_SIZE_IDX]) + continue; + n++; + tot += v; + if (v > mx) + mx = v; + } + + if (peak) *peak = mx; + if (sum) *sum = tot; + if (nproc) *nproc = n; + return n ? 0 : -1; +} + + + +#if defined(HG_MALLOC) && !defined(INLINE_ALLOC) +#include "mem/hg_arena.h" +#include "mem/hg_buddy.h" /* hg_grow_blocked_tick */ +#include "mem/shm_mem.h" /* shm_block, for the grow-blocked gauge */ +#include "evi/evi_core.h" /* EVI_SHM_GROW_BLOCKED_ID */ +#include "evi/evi_modules.h" /* evi_probe/get_params/raise */ + +/* + * The HG_MALLOC idle-cache sweep. + * + * HG_MALLOC's per-thread free caches live in __thread storage, so no other + * process or thread can reach them - a cell parked there is invisible to the + * block accounting and pins a whole block from being reclaimed. The allocator + * flushes its own cache when it is about to grow the arena, but nothing fires + * on a thread that has simply STOPPED allocating, which is precisely what a + * worker does after a traffic burst subsides - the case that strands memory. + * + * So the flush has to be dispatched to each worker to run in its own context, + * exactly the problem signal_pkg_status() above already solves: ipc_send_rpc() + * makes the target execute the job on its own reactor. Two things are carried + * over from it deliberately: + * + * - the self case runs INLINE. Sending ourselves an IPC job would order + * behind the job we are currently running. + * - nothing waits for a result. Its read side blocks on usleep(20) for a + * value that is a request behind; a flush is not a reader, so it is + * fire-and-forget and the next tick simply tries again. + * + * NOT covered, and it needs saying: TCP main's IO pool threads wait on a + * condition variable rather than the reactor, so IPC never reaches them. Their + * caches still need a flag checked at a job boundary. + */ +#define HG_SWEEP_INTERVAL 30 /* seconds; the caches are a slow leak, not a + * fast one, and each sweep costs a lock per + * arena per process */ + +static void rpc_hg_cache_flush(int sender, void *param) +{ + hg_cache_flush_self(); +} + +/* + * The deferred half of GROW-BLOCKED alerting (v3). hg_buddy_grow() latches + * the state under hb->lock and may not raise an event there: + * evi_raise_event() allocates shm, and the arena that just refused to grow + * is the arena it would allocate from. This runs in the timer process with + * no arena lock held, once per sweep - which also gives the design's + * "re-warn interval exceeds a GC cycle" for free. + * + * SHM arena only, and honestly so: each worker's PKG arena is + * MAP_PRIVATE, its grow_blocked visible only inside that process - a pkg + * latch still WARNs in that worker's log and shows in its hg_stats pkg + * section, but no single process can gauge them all. + * + * While the latch holds, the event re-raises every + * HG_GROW_REWARN_SWEEPS sweeps so a subscriber that attached late (or an + * event pipeline that dropped one) still learns of a persistent block. + */ +#define HG_GROW_REWARN_SWEEPS 10 /* x 30s sweep = every 5 minutes */ + +static str hg_gb_arena_str = str_init("arena"); +static str hg_gb_committed_str = str_init("committed_mb"); +static str hg_gb_cap_str = str_init("cap_mb"); +static str hg_gb_refused_str = str_init("grow_refused"); + +static void hg_grow_blocked_event(void) +{ + struct hg_block *hb = (struct hg_block *)shm_block; + static unsigned int blocked_sweeps; + evi_params_p list; + int due, committed_mb, cap_mb, refused; + str arena = str_init("shm"); + + if (!hb) + return; + + /* consume the due flag and sample the numbers under the lock; the + * raise itself must happen outside it */ + lock_get(&hb->lock); + /* promote (or disarm) an armed episode first, so a latch earns its + * event in the same tick that detects it */ + hg_grow_blocked_tick(hb); + /* the profile's proactive grow gate, then the down-slow shrink gate - + * the shm arena's once-per-interval policy heartbeat (pkg arenas tick + * themselves from each process's flush path; a private arena has one + * owner) */ + hg_grow_tick(hb); + hg_shrink_tick(hb); + due = hb->grow_event_due; + hb->grow_event_due = 0; + if (!due && hb->grow_blocked && + ++blocked_sweeps >= HG_GROW_REWARN_SWEEPS) { + due = 1; /* still blocked - re-warn */ + } + if (due) + blocked_sweeps = 0; + if (!hb->grow_blocked) + blocked_sweeps = 0; + committed_mb = (int)(hb->hsize >> 20); + cap_mb = (int)(hb->hcap >> 20); + refused = (int)hb->grow_refused; + lock_release(&hb->lock); + + if (!due) + return; + + if (!evi_probe_event(EVI_SHM_GROW_BLOCKED_ID)) { + LM_WARN("shm GROW-BLOCKED event due, no subscribers - alert " + "on the hg_shm_grow_blocked statistic instead\n"); + return; + } + + list = evi_get_params(); + if (!list) + return; + if (evi_param_add_str(list, &hg_gb_arena_str, &arena) || + evi_param_add_int(list, &hg_gb_committed_str, &committed_mb) || + evi_param_add_int(list, &hg_gb_cap_str, &cap_mb) || + evi_param_add_int(list, &hg_gb_refused_str, &refused)) { + LM_ERR("unable to build the grow-blocked event parameters\n"); + evi_free_params(list); + return; + } + if (evi_raise_event(EVI_SHM_GROW_BLOCKED_ID, list)) + LM_ERR("unable to raise the grow-blocked event\n"); +} + +static void hg_cache_sweep(unsigned int ticks, void *param) +{ + int i; + + hg_grow_blocked_event(); + + /* + * Publish the sweep to threads IPC cannot reach BEFORE dispatching to + * the ones it can. TCP main's IO pool waits on a condition variable + * rather than the reactor, so those threads never receive an RPC job; + * they compare this counter at a job boundary instead + * (hg_cache_flush_if_due()). Bumping it first means a thread that is + * between jobs right now picks the sweep up immediately rather than + * waiting for the next one. + */ + hg_sweep_gen++; + + for (i = 0; i < counted_max_processes; i++) { + if (i == process_no) { + /* never RPC ourselves - see signal_pkg_status() */ + hg_cache_flush_self(); + continue; + } + if (IPC_FD_WRITE(i) <= 0) + continue; + /* fire and forget: a failed dispatch is not worth logging every + * 30 seconds for a process that may simply be shutting down */ + ipc_send_rpc(i, rpc_hg_cache_flush, NULL); + } +} + +int hg_register_cache_sweep(void) +{ + /* + * Both variants, for correctness rather than for a bug that was + * observed: with -a HG_MALLOC the allocator resolves to MM_HG_MALLOC, + * but -a HG_MALLOC_DBG resolves to MM_HG_MALLOC_DBG and would otherwise + * silently decline to register. Every other such test in the tree pairs + * them (mem/shm_mem.c:327, :918, :948, :1033, :1226). + */ + if (mem_allocator_shm != MM_HG_MALLOC && + mem_allocator_shm != MM_HG_MALLOC_DBG && + mem_allocator_pkg != MM_HG_MALLOC && + mem_allocator_pkg != MM_HG_MALLOC_DBG) + return 0; /* not our allocator - nothing caches anything */ + + if (register_timer("hg-cache-sweep", hg_cache_sweep, NULL, + HG_SWEEP_INTERVAL, TIMER_FLAG_SKIP_ON_DELAY) < 0) { + LM_ERR("failed to register the HG_MALLOC cache sweep\n"); + return -1; + } + LM_NOTICE("HG_MALLOC idle-cache sweep registered, every %d s " + "(shm=%s pkg=%s)\n", HG_SWEEP_INTERVAL, + mm_str(mem_allocator_shm), mm_str(mem_allocator_pkg)); + return 0; +} +#else +int hg_register_cache_sweep(void) { return 0; } +#endif /* HG_MALLOC */ int init_pkg_stats(int procs_no) { diff --git a/core_stats.h b/core_stats.h index acccd57fd9b..d2dc2ee09b7 100644 --- a/core_stats.h +++ b/core_stats.h @@ -76,6 +76,25 @@ extern stat_var* slow_msgs; #ifdef PKG_MALLOC int init_pkg_stats(int no_procs); + +/* Registers the HG_MALLOC idle-cache sweep. No-op unless HG_MALLOC is the + * live allocator. Must be called PRE-FORK: register_timer() only accepts + * registrations before the timer processes are created. */ +int hg_register_cache_sweep(void); + +/* + * Highest pkg high-water reached by ANY process, the sum across all of them, + * and how many reported. pkg arenas are private memory, so a process cannot + * read another's directly - this walks the shared pkg_status array that the + * pkmem: statistics are built from, refreshing it by IPC first. + * + * Needed because the -M advisory has to be sized for the WORST process, and + * the one answering an MI command is typically the idlest (over mi_fifo it is + * the FIFO listener, which does no SIP work at all). + * + * Returns 0 on success, -1 if no process has reported yet. + */ +int hg_pkg_peak_all(unsigned long *peak, unsigned long *sum, int *nproc); #endif #endif /*STATISTICS*/ diff --git a/evi/evi_core.c b/evi/evi_core.c index 909de2dbbcc..11f1a73f9c8 100644 --- a/evi/evi_core.c +++ b/evi/evi_core.c @@ -40,6 +40,11 @@ static str evi_core_table[] = { CORE_EVENT_STR(PKG_THRESHOLD), CORE_EVENT_STR(PROC_AUTO_SCALE), CORE_EVENT_STR(TCP_DISCONNECT), + /* Published unconditionally so event_route[] can always subscribe; + * only ever RAISED when the shm allocator is HG_MALLOC with a growth + * cap and a latched resource refusal (core_stats.c, the sweep timer). + * An event that exists but never fires costs one table entry. */ + CORE_EVENT_STR(SHM_GROW_BLOCKED), }; int evi_register_core(void) diff --git a/evi/evi_core.h b/evi/evi_core.h index 3dd25303285..f265ee515f0 100644 --- a/evi/evi_core.h +++ b/evi/evi_core.h @@ -34,6 +34,11 @@ #define EVI_PKG_THRESHOLD_ID 3 #define EVI_PROC_AUTO_SCALE_ID 4 #define EVI_TCP_DISCONNECT 5 +/* carries the same #ifdef STATISTICS fragility as SHM_THRESHOLD above + * (see the FIXME) - with statistics off, the table skips an entry and + * every later hardcoded id is one too high. The raise site is itself + * statistics-gated, so the pairing stays consistent. */ +#define EVI_SHM_GROW_BLOCKED_ID 6 #define EVI_CORE_PREFIX "E_CORE_" diff --git a/globals.c b/globals.c index bf8200c43d8..eb6b84418e7 100644 --- a/globals.c +++ b/globals.c @@ -64,6 +64,14 @@ int udp_workers_no = UDP_WORKERS_NO; char *udp_auto_scaling_profile = NULL; /* if the auto-scaling engine is enabled or not - this is autodetected */ int auto_scaling_enabled = 0; + +/* v3 elastic-arena knobs - see globals.h for ownership and timing */ +unsigned long hg_shm_cap_bytes = 0; +unsigned long hg_pkg_cap_bytes = 0; +char *hg_shm_profile_name = NULL; +char *hg_pkg_profile_name = NULL; +int hg_ram_floor_mb = 0; +int hg_autoscale_dry_run = 0; /* auto-scaling sampling and checking time cycle is 1 sec by default */ int auto_scaling_cycle = 1; /*!< by default choose the best method */ @@ -95,6 +103,13 @@ int check_via = 0; /* debugging level for memory stats */ int memlog = L_DBG + 11; int memdump = L_DBG + 10; +/* pin each worker to one CPU (0 = off). See pt.c pin_worker_to_cpu(). */ +int pin_workers = 0; +/* optional per-process-type CPU lists, e.g. "0-7". NULL = all allowed CPUs */ +char *pin_udp_cpus = NULL; +char *pin_tcp_cpus = NULL; +char *pin_timer_cpus = NULL; +char *pin_module_cpus = NULL; unsigned int shm_memlog_size = 0; /* debugging in case msg processing takes. too long disabled by default */ int execmsgthreshold = 0; diff --git a/globals.h b/globals.h index 02015426ed1..5f0e8827edf 100644 --- a/globals.h +++ b/globals.h @@ -55,6 +55,19 @@ extern int udp_workers_no; extern char *udp_auto_scaling_profile; extern enum poll_types io_poll_method; extern int auto_scaling_enabled; + +/* v3 elastic HG_MALLOC arenas. Defined in main.c UNCONDITIONALLY (the -m/-M + * parser and cfg.y reference them in every build); consumed only when the + * HG allocator is selected - inert otherwise. Caps come from the command + * line (-m INIT:CAP) because the reservation precedes config parsing; the + * profile names/floor/dry-run come from the config and are applied + * post-parse. */ +extern unsigned long hg_shm_cap_bytes; +extern unsigned long hg_pkg_cap_bytes; +extern char *hg_shm_profile_name; +extern char *hg_pkg_profile_name; +extern int hg_ram_floor_mb; /* 0 = auto: max(256MB, MemTotal/20) */ +extern int hg_autoscale_dry_run; /* 1 = ticks log, never act */ extern int auto_scaling_cycle; /* TCP network layer related parameters */ @@ -121,6 +134,11 @@ extern int is_pre_daemon; extern int memlog; /*!< debugging level for printing memory debugs */ extern int memdump; /*!< debugging level for dumping memory status */ +extern int pin_workers; /*!< pin each worker process to a single CPU */ +extern char *pin_udp_cpus; /*!< CPU list for UDP workers */ +extern char *pin_tcp_cpus; /*!< CPU list for TCP workers */ +extern char *pin_timer_cpus; /*!< CPU list for timer processes */ +extern char *pin_module_cpus; /*!< CPU list for module processes */ extern unsigned int shm_memlog_size; extern int execmsgthreshold; /*!< Maximum number of microseconds a SIP msg processing can last before triggering Warning log */ diff --git a/help_msg.h b/help_msg.h index 7eb4544e518..c9c9f42e455 100644 --- a/help_msg.h +++ b/help_msg.h @@ -96,6 +96,7 @@ Options:\n\ F_PARALLEL_MALLOC F_PARALLEL_MALLOC_DBG\n\ Q_MALLOC Q_MALLOC_DBG\n\ HP_MALLOC HP_MALLOC_DBG\n\ + HG_MALLOC HG_MALLOC_DBG\n\ -k allocator The pkg memory allocator to use (overrides -a)\n\ -s allocator The shared memory allocator to use (overrides -a)\n\ -e allocator The restart-persistent memory allocator to use (overrides -a)\n\ diff --git a/ip_addr.h b/ip_addr.h index 0ed2dcb7b3d..ac7ffa44c10 100644 --- a/ip_addr.h +++ b/ip_addr.h @@ -154,6 +154,7 @@ struct socket_id { int proto; int port; int workers; + char* pin_cpus; int tos; enum si_flags flags; struct socket_id* next; diff --git a/main.c b/main.c index c0684745f47..3f8b0d444ae 100644 --- a/main.c +++ b/main.c @@ -420,6 +420,26 @@ int main(int argc, char** argv) switch(c){ case 'M': pkg_mem_size=strtol(optarg, &tmp, 10) * 1024 * 1024; + /* + * -M INIT[:CAP] - CAP megabytes of growth headroom for + * the v3 elastic arena. A COMMAND LINE extension rather + * than a config global because this runs before the + * config is parsed, and the reservation (for tier-1 + * hugetlb, a map-time pool reservation) must exist + * before the first arena is created. The + * auto-scaling profile from the config supplies the + * POLICY within this reservation, never the + * reservation itself. + */ + if (tmp && *tmp == ':') { + hg_pkg_cap_bytes = + strtol(tmp + 1, &tmp, 10) * 1024 * 1024; + if ((unsigned long)pkg_mem_size > hg_pkg_cap_bytes) { + LM_ERR("-M cap smaller than the initial " + "size: %s\n", optarg); + goto error00; + } + } if (tmp &&(*tmp)){ LM_ERR("bad pkgmem size number: -m %s\n", optarg); goto error00; @@ -427,6 +447,16 @@ int main(int argc, char** argv) break; case 'm': shm_mem_size=strtol(optarg, &tmp, 10) * 1024 * 1024; + /* -m INIT[:CAP], see the -M note above */ + if (tmp && *tmp == ':') { + hg_shm_cap_bytes = + strtol(tmp + 1, &tmp, 10) * 1024 * 1024; + if ((unsigned long)shm_mem_size > hg_shm_cap_bytes) { + LM_ERR("-m cap smaller than the initial " + "size: %s\n", optarg); + goto error00; + } + } if (tmp &&(*tmp)){ LM_ERR("bad shmem size number: -m %s\n", optarg); goto error00; diff --git a/mem/README.hg_arena_v2 b/mem/README.hg_arena_v2 new file mode 100644 index 00000000000..a173c62a6d9 --- /dev/null +++ b/mem/README.hg_arena_v2 @@ -0,0 +1,319 @@ +HG_MALLOC arena v2 - reclaiming the carve +========================================= + +DESIGN ONLY. Nothing in this file is implemented yet. It exists so the +reasoning is reviewable before any code is written, and so the measurements +that motivate it are not lost. + + +The problem, measured +--------------------- + +From core:hg_stats on a production gateway carrying real traffic: + + carved 53.42 MB + live_committed 9.14 MB + slab_recycled 44.28 MB <- 83% of the carve, idle + cells carved 154,454 + cells live 1,353 <- 0.88% in use + +Chunks are carved per size class and never returned, so a burst pins its peak +for the lifetime of the process. Class 96 holds 6 MB across 24 chunks; class +24576 holds 11.7 MB - a quarter of the entire carve - from 500 cells. + +This, and not size-class round-up, is the origin of the "budget about twice +what F_MALLOC needs" rule that forces -m 5120 and a 2880-page hugetlb pool on +these nodes. Fixing it retires that rule. + +A dead end, measured and rejected: the 32-byte cell header (24 bytes of it +from DBG_MALLOC being compiled in). Real, but worth only ~32 KB of 8.4 MB +here, because the average allocation is ~6 KB. Not worth losing the _DBG +allocators for. + + +Two constraints that shape everything +------------------------------------- + +1. Live blocks can never be relocated. Callers hold raw void *, there is no + handle indirection and no pointer map. This is therefore a garbage + collector, never a compacting defragmenter. The only sanctioned relocation + channel is hg_realloc(), which returns a new pointer by contract. + +2. The carve is irreversible. hg_chunk_backing() is a monotonic atomic add on + hb->hoff, and on exhaustion deliberately leaves hoff past the end because + undoing it would race other bumpers. Nothing returns to the arena, so + reclaim must mean re-cutting inside what is already carved. + + +Structure +--------- + + arena (-m, reserved, mlocked and pre-faulted at init) + -> 2 MB hugepages, each the top order of a buddy tree + -> buddy blocks, 8 KB .. 2 MB, split and merged on demand + -> each block dedicated to one cell class + -> cells, per-thread LIFO caches + +Only the bottom level is on the hot path. Everything above it lives in the +slow path that already takes hb->lock, which is how the allocator's measured +2.73% self-time survives the change. + +Splitting is pure bookkeeping: the memory is already mapped and pre-faulted, +so a 16 KB block becomes two 8 KB blocks by changing records alone. Merging +two free buddies is the reverse, and is the only defragmentation that is legal +here, because it operates on free space rather than live objects. + +A page is not one block per order. Three 8 KB blocks simply means splitting +twice; a page ends up as e.g. 8 + 8 + 8 + 8(free) + 16 + 32. Buddy constrains +only that a size is a power of two multiple of MIN_ORDER, that a block is +naturally aligned to its own size, and that it merges only with the half it +was split from. The count per order is unconstrained. + +Top order equals page size, so the hierarchy terminates naturally: there is no +cross-page merging to implement, and a wholly free top block is exactly one +huge page. + + +Address to block +---------------- + + page = (addr - hbase) >> 21 + order = leaforder[(off & (2MB-1)) >> 13] one byte per 8 KB leaf + base = mask by order + +hbase is 2 MB aligned, so this holds by construction. PREREQUISITE: blocks +must be aligned at carve time. hg_chunk_backing() today aligns to 64 bytes +only, so the mask does not work as-is. + + +Reclaim: two mechanisms that are useless apart +---------------------------------------------- + +GC un-types. A block whose cells are all free is returned to the buddy pool +and stops belonging to a size class. This is what recovers the 44 MB: a block +full of free 96-byte cells is worthless to a class-768 requester until it is +returned and re-cut. + +Defrag re-joins. A freed block merges with its buddy, that with its buddy, and +so on up to a whole page. O(1) per step, since a buddy's address is the block's +own with one bit flipped. Its one limitation is inherent: merging is +buddy-only, not any-adjacent, so two free neighbours that are not partners +never merge. + +Run only GC and the result is correctly-sized but fragmented free space. Run +only defrag and nothing ever becomes free enough to merge. + +Both are event-driven, not periodic. A free pushes onto a thread's LIFO and +never touches the block; the block's live counter only changes at cache/block +transitions, which is 1-3% of operations and already under the lock. Checking +"did this block hit zero, and can it merge" there costs almost nothing and +returns memory the moment it is genuinely free - which matters, because under +memory pressure a deferred scheme is slowest exactly when the reserve is most +needed. + +Eager merging can thrash at a boundary (free, merge to 16 KB, immediately need +8 KB, split back). Delayed coalescing fixes it: keep a few free blocks per +order deliberately unmerged as hysteresis. + +A periodic sweep survives for one job only: flushing the caches of threads +that have gone idle. No event fires on a quiet thread, so its cached cells are +unreachable by any inline mechanism. + + +Allocation policy decides how much is reclaimable at all +-------------------------------------------------------- + +GC and defrag are passive - they act on blocks that happen to empty. What +causes blocks to empty is policy: + + - concentration: when a class needs a new current block, take the FULLEST + partial, never the emptiest and never round-robin. Without this, refill + keeps topping up every block, live cells stay smeared, and the sweep finds + nothing. + + - thread ownership: a thread claims a whole block and carves cells locally. + Beyond the reduction in lock traffic, a block owned by one thread drains + as a unit instead of holding one stray survivor from each of sixteen. + + - splitting: always split the smallest free block that fits, so large free + blocks are preserved by construction. + + - page preference: satisfy small requests from pages that already have a + free block of that order, or are already fragmented; satisfy large + requests from untouched pages. Keep the mess concentrated, or every page + ends up lightly speckled with survivors and none can serve a large block. + +The reclaim machinery is the easy half. Policy is what determines whether this +recovers 40 MB or 2 MB. + + +Cross-thread frees +------------------ + +Ownership cannot assume the owner frees: hg_used() clamps at zero precisely +because slots go negative when one process frees another's cells. A cell freed +by a non-owner goes on a per-block remote-free list, reconciled when the owner +next touches the block or when the sweep runs. + +This is the one place a write-back queue is correct - the deferred work is +genuinely more expensive than the deferral, because doing it inline would mean +touching another thread's structures. + + +Cache budget +------------ + +The current HG_PRIVATE_MAX is a flat count of 256 cells per class regardless +of size, which for class 65536 permits a 16 MB hoard per process. It bounds +nothing. The cap belongs in bytes. + +The budget must also be derived rather than fixed: + + per_thread_budget ~= arena_size * F / concurrent_consumers F ~ 10-25% + +The binding case is not shm but pkg. pkg is MAP_PRIVATE per process, and TCP +main is one process running an IO thread pool - sixteen threads sharing one +8 MB pkg arena. At 21 classes and one 8 KB block each, caches alone would +consume 5.4 MB of that 8 MB. On shm the same rule is comfortable. This +constraint did not exist before 4.1, because there was one thread per process. + + +Metadata +-------- + +Per 2 MB page: leaf-order array 256 B, buddy free bitmap 64 B (511 bits covers +all nine orders), page descriptor 64 B. 384 B per page. + + pkg -M 8 4 pages 1.5 KB 0.018% + shm -m 256 128 pages 48 KB 0.018% + shm 5 GB 2560 pages 960 KB 0.019% + +Plus a flat ~1.3 KB fullness index. The GC needs no structure of its own; the +empty-page queue threads through page descriptors that already exist. + +Per-block metadata (class, live count, links, remote-free head) is ~32 B. As a +separate array that would be 20 MB on a 5 GB arena - twenty times everything +else combined. Put it in the first 32 bytes of the block itself: zero external +cost, better locality, and one cell fewer per block. + +Metadata is carved from the front of the arena rather than given its own page. +It then inherits the tier the ladder already achieved, is shared for shm and +private for pkg without a decision, and costs no extra hugepages. A dedicated +2 MB page would be 25% overhead on an 8 MB pkg arena, and 30 processes each +wanting one would burn 60 MB to hold 45 KB. + + +Failure mode to design against +------------------------------ + +Commit lock: the arena fully committed with a stray survivor in every block, +so a large request cannot be served though the arena is mostly free. Splitting +does not cure this - there is nothing free to split. Only draining does. + + - keep a floor of free pages in reserve, not consumable by an expanding + class; + - trigger the sweep on reserve pressure, not only on idle time. + + +Rejected +-------- + + - compacting defragmentation: impossible, see constraint 1. + - whole-chunk reclaim of today's fixed 256 KB chunks: too coarse. Class 96 + packs 2,730 cells per chunk, so one survivor pins 256 KB and reclaim would + almost never fire. + - a page dedicated to a size CLASS: the floor becomes 21 * 2 MB = 42 MB, + worse than what is being reclaimed. Blocks, not pages, are class-dedicated. + - a general write-back queue for GC/defrag: the deferred work is already + nearly free, the queue needs storage inside an allocator, a full queue is a + new failure mode, reclaim latency peaks exactly under pressure, and + deferred entries need revalidating against a structure that keeps mutating. + - per-CPU arena sharding, for now: NUMA was A/B'd on this hardware and remote + CPU with local memory came out marginally FASTER, and CPU pinning measured + a wash at 12 workers on 16 cores. Build the seams, ship at N=1, raise it + only if a profile shows lock pressure. Today no lock symbol appears above + 0.01%. + + +Acceptance +---------- + + - speed: no regression on the three-arm bench harness, three validated reps + per arm. + - memory: core:hg_stats before and after on a real node; slab_recycled should + collapse from ~83% of carve. + - correctness: the hgstress soak, which already catches double hand-out by + stamping every word with a pid-derived value. + + +Reaching per-process pkg memory +------------------------------- + +A pkg arena is MAP_PRIVATE, and after the move to __thread free-stacks a +thread's cached cells live in TLS. Neither is reachable from any other process +or thread, so there can be no central sweeper: a flush must be executed by the +owning thread. + +The core already solves this shape. Despite its name, signal_pkg_status() in +core_stats.c does not signal - it uses ipc_send_rpc() to make the target run +rpc_get_pkg_stats() in its own context, writing results into the shared +pkg_status[] array. That is the mechanism to reuse. + +Its read side, however, should not be copied. The caller reads pkg_status[] +immediately after dispatching, so it returns the previous request's value; the +only synchronisation is usleep(20), which is 20 microseconds for a process to +wake from epoll and write back; and it is rate limited to once per second per +process. The published pkg figures are therefore stale by up to a second. + +That flaw does not affect the sweep, because staleness is a reader problem and +a cache flush is not a reader. Dispatch fire and forget, never wait for a +result, and re-check the reserve on the next tick. Two things do carry across: + + - a process must flush its own cache inline, never by RPC to itself, or it + orders against its own pending job; + - promptness is not guaranteed. A busy worker may not reach its reactor for + some time, so the reserve floor needs enough hysteresis that a slow flush + is not an allocation failure. + +TCP main's IO threads are the exception: they wait on a condition variable +rather than the reactor, so IPC does not reach them. They need a flag checked +at job boundaries. + +No dedicated GC or defrag worker is needed, and none should be added. The +mechanisms are inline on whichever thread is already allocating, and the sweep +is a list walk measured in microseconds, seconds apart. + + +Instrumentation: counters for rates, events for transitions +----------------------------------------------------------- + +At 800 CPS the allocator sees on the order of 10^5 cell operations per second. +Cache/block transitions are 1-3% of that. Block returns are burstier still - +near zero while load is steady, since a stable working set keeps blocks owned +and absorbs the churn in caches, then a flurry as caches drain afterwards. + +So there must be no per-block or per-merge event. They are individually +meaningless, and an EVI event with an event_route subscriber executes SIP +script: at 10^3 per second that would cost more than the allocator it reports +on. Even unsubscribed, the dispatch is not free. + +Counters carry the rates: blocks returned, blocks re-cut, merges, splits, cells +flushed, sweep runs - alongside the existing carved, slab_recycled and +live_cells. + +Per-process pkg counters are constrained by the section above: they can only be +read through the stale, rate-limited pkg_status path. Either aggregate them +into shm as they are produced, or publish them with an explicit staleness +caveat. shm counters have no such problem. + +Three events, all low rate by construction: + + - a whole page freed or re-graded. All 256 blocks must drain first, so this + is single digits per minute even during churn. It shows reclaim working. + - the reserve floor crossed, in either direction. The valuable one: it + precedes allocation failure, hysteresis bounds its rate, and it is the only + one an operator can act on in time. + - allocation failed, from commit lock or exhaustion. Rare and critical. + +This matches the module precedent - cachedb_perf carries four EVI events for +genuine state changes and uses statistics for everything rate-shaped. diff --git a/mem/README.hg_malloc b/mem/README.hg_malloc new file mode 100644 index 00000000000..f575f6e2dbf --- /dev/null +++ b/mem/README.hg_malloc @@ -0,0 +1,472 @@ +HG_MALLOC - hugepage-backed slab allocator +========================================== + +HG_MALLOC is one of the memory allocators OpenSIPS can use for its shared +(shm) and process-private (pkg) memory. It is built around two ideas the +other allocators do not use: huge pages, and a per-process allocation fast +path that takes no lock and touches no shared cache line. + +It is selected at runtime like any other allocator: + + opensips -a HG_MALLOC # both shm and pkg + opensips -s HG_MALLOC # shm only + opensips -k HG_MALLOC # pkg only + +and compiled in by default (-DHG_MALLOC in Makefile.conf). "opensips -V" +lists it in the flags line when present, and "opensips -h" lists the +accepted values. + + +Sizing +------ + + -m shared memory, one arena for the whole server + -M private memory, per process + +Minimums for a small configuration (a handful of modules): + + absolute minimum -m 8 -M 2 + recommended floor -m 64 -M 16 + +Below the minimum the allocator refuses to serve and says so, e.g. + + ERROR:core:carve_chunk: shm: no more HG_MALLOC arena memory for a + 65536 byte chunk (class 7) - increase the arena size + +A real configuration needs more. Loading cachedb_perf, for instance, +raises the shm floor noticeably: at 200 CPS, -m 64 produced a stream of +carve_chunk failures and tm could not clone messages, while -m 256 ran +clean. If you see carve_chunk in the log, raise -m or -M; it is not a +leak. + +Note that -m is rounded up to a whole number of huge pages, so on a +machine with 2 MB huge pages the real granularity is 2 MB. + + +Sizing it against the other allocators +-------------------------------------- + +*The old "budget twice F_MALLOC" rule is RETIRED - see below. It is kept +here because the reasoning still explains what to watch for, and because +the number is quoted in older notes.* + +An arena that is enough under one of the other allocators may not be +enough here, and the gap widens with call concurrency rather than with +call rate. Two properties of the design caused it: + + * every allocation is rounded up to its size class, so a request lands + in a cell up to one class larger than it asked for - this STILL + applies, and it is the part you still budget for; + * a freed cell went back to the class it came from and was never + reclassified, so memory the peak of one class took was not available + to another afterwards - this is FIXED in v2. A block whose cells have + all been freed is un-typed, returned to the buddy allocator, and can + be re-cut for any other class. See README.hg_arena_v2, which names + exactly this as the origin of the 2x rule and fixing it as what + retires the rule. + +Neither is a leak, and neither shows up at low concurrency - both are +proportional to how many objects are alive at once. + +Measured on v1, on a topology-hiding proxy (three nodes, four UDP workers +each, INVITE/200/ACK/BYE held for 500 ms, so roughly 1500 calls in flight +per node at 3000 CPS): + + F_MALLOC -m 512 clean + HG_MALLOC -m 512 carve_chunk failures on the hot class within + seconds; topology_hiding could no longer store + its state and the calls were lost + HG_MALLOC -m 1024 clean - 0 carve_chunk, 0 failed calls, and + 25% less CPU than F_MALLOC at the same load + +Under v2 the reclaim closes most of that gap. On the production billing +gateways HG_MALLOC peaks at 14.8 MB of shm while Q_MALLOC in the same +role peaks at 16.1 MB - lower, while carrying four times the traffic. So +do not apply a multiplier: size from measurement. Run the workload and +ask hg_advise, which derives -m and -M from the high-water marks the +allocator actually saw, and confirm against the log rather than by +arithmetic. The requirement differs with the mix of allocation sizes a +particular config makes. + +The failure is loud but it is not fatal, which is the trap: the server +keeps running and keeps answering, while the module that could not get +memory quietly drops what it was doing. Watch for carve_chunk in the log +of any new deployment before trusting a capacity number from it. + +Unlike the other allocators, HG_MALLOC pre-faults and mlock()s its whole +reservation at startup, so the memory is resident immediately rather than +growing as it is used. Budget for -m plus (-M x number of processes). + + +Huge pages +---------- + +The arena is obtained through a four-tier ladder, best first. Each tier is +attempted and then *verified* through /proc - never assumed from the +kernel version or from sysfs settings: + + 1 mmap(MAP_HUGETLB) huge pages from the hugetlb pool + 2 MADV_HUGEPAGE THP, huge at first fault + 3 MADV_COLLAPSE THP, retrofitted after filling + 4 plain 4K pages always works + +A miss degrades to the next tier; it never fails the startup. Which tier +was actually achieved is logged, and that log line is the only reliable +way to know: + + NOTICE:core:hg_malloc_init: shm HG_MALLOC arena: 256 MB on + MAP_HUGETLB 2M pages, 256 MB pinned + +Tier 1 needs a hugetlb pool big enough for the arena, e.g. + + sysctl -w vm.nr_overcommit_hugepages= + +Huge pages are worth having but are not where most of the benefit comes +from. Measured on one host at 800 CPS, allocator CPU self-time varied only +between 2.75% (tier 1) and 3.38% (tier 4, huge pages fully disabled) - +while the other allocators on the same machine and workload ranged from +9.5% to 22.8%. If the machine cannot give you huge pages, HG_MALLOC is +still worth using. + + +Carving, and what the memory statistics mean +-------------------------------------------- + +HG_MALLOC does not treat the arena as one pool of interchangeable bytes. +It CARVES it: the first time a process needs a cell of some size class, a +whole chunk is cut out of the arena, divided into equal cells of that +class, and given to that process. A chunk is never handed back. Freeing a +cell returns it to a per-process free stack, ready to be reused - but only +for the same size class, and preferentially by the same process. + +That single property is why the memory statistics read differently here +than under q_malloc or f_malloc, where a freed fragment returns to a +general pool and can serve a request of any size. + + used_size payload bytes currently handed to callers. Rises and + falls with load; same meaning as everywhere else. + + real_used_size live commitment - the cell slots currently handed + out, including each cell's header and the round-up to + its size class. Rises and falls with load. + + max_used_size high-water mark of real_used_size. Climbs to the + working set and then holds, as for the other + allocators. + + free_size arena not yet carved. This is the figure that + predicts "no more HG_MALLOC arena memory": it counts + down as chunks are cut, and never goes back up. + + fragments live cells. + +Two consequences are worth stating plainly. + +First, free_size + real_used_size does not add up to total_size. Under +q_malloc it does. Here the difference is the carved-but-idle capacity - +cells sitting on a free stack. They are genuinely reusable, so calling +them "used" would overstate demand; they are also already committed to one +size class, so calling them "free" would overstate what is available. The +identity that does hold is: + + free_size + carved = total_size + +Second, recycled capacity is not interchangeable. An arena can report a +large free_size and still fail an allocation, if what has been carved does +not match the size being asked for. In practice a given workload settles +into a stable mix of sizes and this does not arise, but it is the reason +free_size is defined as "not yet carved" rather than "not in use". + +None of this has to be inferred: hg_stats below reports each quantity +separately. + + +hg_stats +-------- + + opensips-cli -x mi hg_stats + +Reports the allocator's own view of each arena it manages: shm, and the +pkg arena of whichever process answered the command. pkg arenas are +per-process, so that half is one process's arena, not a server-wide total. + + tier which of the four huge-page tiers the arena got + total_size arena size, as given by -m / -M + pinned_mb MB protected from swapping + carved bytes ever cut out of the arena into chunks + carved_peak high-water mark of the above + chunks how many chunks that is + free_to_carve total_size - carved, i.e. shmem:free_size + live_cell_bytes cell slots currently handed out + live_payload of which, requested payload, i.e. shmem:used_size + live_cells number of live cells, i.e. shmem:fragments + live_peak high-water mark, i.e. shmem:max_used_size + recycled carved but idle: on a free stack or in the global + pool + classes per size class: cell size, chunk count, total cells + +The per-class breakdown is the quickest way to tell whether an arena is +being carved into the sizes the workload actually asks for. + + +Per-module memory accounting +---------------------------- + +Building with -DSHM_EXTRA_STATS lets a config attribute shared memory to +named groups of modules: + + mem-group = "core": "core" + mem-group = "tm": "tm" + +which then appear as ordinary statistics - shmem_core:memory_used, +shmem_tm:fragments and so on - queryable over MI. + +It is not free, but close to it. Measured on the same host and workload as +above, HG_MALLOC's allocator CPU self-time went from 2.74% without it to +2.90% with it (two runs each). The symbols that exist only in this build - +update_module_stats plus the two per-cell index accessors - account for +0.13-0.15% of CPU directly. Most of that is not specific to this +allocator: update_module_stats is called from the generic shared-memory +wrappers, so every allocator pays it. + +Set against the 9.5% to 22.8% the other allocators spend on allocation +alone, the accounting is worth having. + + +Sharing the hugetlb pool with cachedb_perf +------------------------------------------ + +cachedb_perf reserves its own huge-page arena (arena_hugepage_mb). Both it +and HG_MALLOC can be on tier 1 at the same time, but they do not +coordinate with each other - each simply asks the kernel and takes what is +available. Two consequences: + + * Size the pool for the SUM of every arena, not the largest one: + + nr_overcommit_hugepages >= (-m + -M + arena_hugepage_mb) / 2 + + on a 2 MB huge page machine, plus headroom. + + * HG_MALLOC always claims first. shm and pkg are initialised in main() + before any module is loaded, so cachedb_perf only ever sees what is + left. With a pool too small for both, it is the *core* arena that + degrades to a lower tier while cachedb_perf still gets tier 1. + +Nothing fails when the pool is short - each arena just lands on a lower +tier - so check both startup log lines rather than assuming. + +Also note that pkg is a private mapping made before the workers fork, so +every worker copies its huge pages as it writes to them; pool consumption +grows with the number of processes. + + +CPU pinning +----------- + +The core option "pin_workers=1" pins each worker process to a single CPU, +round-robin. It is off by default and independent of the allocator, but it +matters here: HG_MALLOC's fast path keeps its free lists in per-process +memory, so once workers stop migrating those lists stay warm in one CPU's +cache. + +Each kind of process can instead be given its own CPU list, which is +usually what you want - the point of pinning is normally to keep the +packet-receiving workers away from everything else: + + pin_udp_cpus = "0-7" + pin_tcp_cpus = "8-11" + pin_timer_cpus = "12" + pin_module_cpus = "13-15" + +Setting any of these turns pinning on by itself, so pin_workers=1 is only +needed to pin everything with no grouping. A type with no list may use +every CPU it is allowed. Lists take the usual "0-7,12" form. + +A single UDP listener can also name its own list, which overrides the group +for just that listener's workers: + + pin_udp_cpus = "8-11" + socket = udp:203.0.113.10:5060 use_workers 2 pin_cpus "0-1" + socket = udp:10.0.0.10:5060 use_workers 4 + socket = udp:10.0.0.10:5080 use_workers 2 + +Here the two workers of the first listener are confined to CPUs 0-1, while +the other six fall back to pin_udp_cpus and are spread across 8-11 by +occupancy. A listener that names no list is unaffected, so the coarse +option stays the sensible default and pin_cpus is the exception you make +for the interfaces that need one - typically to keep a public-facing +listener off the cores serving the internal ones. + +A list shorter than use_workers is legitimate, not an error: the workers +share it by occupancy, so three workers on one CPU is a valid thing to ask +for. Only UDP-based protocols fork per-listener workers, so pin_cpus on a +TCP listener is reported and ignored (use_workers on TCP is already a +no-op for the same reason). + +There is no bin or hep group. Neither forks processes of its own: they are +transport protocols carried by the ordinary UDP and TCP workers, so a +"bin:" listener is served by TCP workers and follows pin_tcp_cpus, and HEP +follows the UDP or TCP list depending on the transport it runs over. + +CPUs are chosen from the set the process is already allowed to run on, so +a cpuset or container restriction is respected and can only be narrowed, +never widened - a group is intersected with that set, never substituted +for it. If the intersection is empty those processes are left unpinned +with a warning rather than failing the startup, and a malformed list is +reported naming the option and the value. + +Within its group, each process gets the least-occupied CPU, counted over +the processes running at that moment rather than derived from a process +index. This matters with auto-scaling: workers come and go, and an index +would say nothing about who is already running where. + +NUMA placement is deliberately NOT implemented. On a 2-socket test machine +the arena was pinned to one node and the workers run first on the same node +and then on the other; the remote arrangement measured no worse (in fact +marginally better, the node hosting the network interrupts being the busier +one). Until a workload shows a real cost, there is nothing here to recover. + +Limitations +----------- + + * Allocations are served from fixed size classes up to 64K, and from a + coalescing tier above that. v2 does garbage-collect and defragment - + a fully drained block loses its class and merges with its buddy - but + reclaim is WITHIN the arena: the reservation is never handed back to + the OS while the server runs, by design, because it is pre-faulted + and mlock()ed. + * Reclaim needs a block to drain COMPLETELY, so a class whose blocks + each keep one live cell holds all of them, and merging is buddy-only, + so two free neighbours that are not buddies never merge. + * On exhaustion it fails loudly rather than falling back to another + allocator. Raise -m / -M. + * NUMA-aware arena placement is not implemented - see CPU pinning + above for why. + * The arena is excluded from core dumps (MADV_DONTDUMP) by default. Since + every page is resident, including it would add the full arena size to + every worker's core file. + + Set HG_DUMP_ARENA=1 in the environment to include it instead. Needed + whenever the allocator itself is under suspicion: VM_DONTDUMP overrides + coredump_filter, so no filter value can recover the arena, and without it + a core has no free lists, no class counters and no cell headers - gdb + simply reports "Cannot access memory" for shm_block. It is an environment + variable rather than a config parameter because it must apply during + allocator init, before the config file is read. Budget the full arena + size per crashing worker, and remember coredump_filter must also carry + bit 6 (shared huge pages, 0x40) - the kernel default 0x33 does not. + + +Testing +------- + +modules/hgstress is a throwaway stress module that hammers shm from every +worker at once and detects the failure mode ordinary tests miss - the same +block being handed to two processes. Each worker stamps its blocks with a +value derived from its own pid and re-reads them later, so a foreign +writer is caught and named. See that module's source for the parameters. + + +The v3 elastic arena +-------------------- + +From v3 the arena distinguishes COMMITTED memory (pre-faulted, pinned, +usable) from the RESERVATION around it, and can move the committed size in +both directions at runtime. With no cap configured, nothing changes: the +arena is fixed at -m/-M exactly as in v2. + +Giving it room - the command line, not the config: + + opensips -m 128:1024 -M 16:64 ... + +reserves a 1024 MB shm span (and a 64 MB span per pkg arena) while +committing only the 128/16 the run starts with. The cap lives on the +command line for a hard reason: the shm arena is created BEFORE the config +file is parsed, and on the hugetlb tier the whole cap is reserved from the +page pool at map time - there is no later moment at which a config value +could still shape the reservation. On tiers 2-4 the uncommitted remainder +costs only page-table entries; on tier 1 it is EARMARKED POOL, see sizing +below. + +Growth is exhaustion-triggered by default: an allocation that would have +failed instead commits one granule (16 MB) more and retries, up to the +cap. Every delta re-negotiates its backing with the kernel and is +VERIFIED, never assumed - a THP arena's delta may land on 4K pages, and +hg_stats then reports the split in tier_bytes rather than pretending one +tier. Growth is refused, never degraded, when the host cannot back it: +the mlock that populates the delta reports failure as a clean errno, and +the host-RAM check before it keeps MemAvailable above a floor of +max(256 MB, MemTotal/20), overridable with + + hg_ram_floor_mb = 2048 + +pkg deltas are charged times the process count in that check - every +worker grows its own private arena under the same workload. + +Shrink returns whole-free TOP pages (madvise, so every mapped process +sees it; on tier 1 the pages go back to HugePages_Free) after a +sustained quiet period, never below the starting -m/-M unless a profile +says otherwise. The top-order free list is kept address-ordered so +carves concentrate low and the top drains. + +Policy - the worker autoscaler grammar, reused: + + auto_scaling_profile = MEM_SHM + scale up to 1024 on 80% for 3 cycles within 10 + scale down to 256 on 30% for 120 cycles + + shm_auto_scaling_profile = MEM_SHM + pkg_auto_scaling_profile = MEM_PKG + +"up to"/"down to" are MB (rounded to whole huge pages); percentages are +carved-of-committed; a cycle is one sweep interval (30 s). A profile +adds PROACTIVE growth - the arena grows when usage holds over the +up-threshold, before any allocation fails (the exhaustion path stays +armed for bursts between cycles) - and makes the down-threshold and +cycle count drive the shrink gate. The grammar's implicit 10x delay +after a scale-down is honoured as a post-grow cooldown. The scale-up +target becomes the ceiling WITHIN the -m cap; the scale-down target +becomes the shrink floor and may sit below -m: with a profile attached, +the profile is the ask and -m is just the starting size. A profile that +names nothing, exceeds the reservation, could never act, or floors +below two huge pages FAILS STARTUP - a policy that cannot work is a +misconfiguration, not a default. The pre-fork parent pkg arena predates +the config and stays fixed; every per-child pkg arena follows the +profile. + + hg_autoscale_dry_run = 1 + +keeps every decision advisory: ticks and even emergency growth log what +they WOULD have done ("DRY RUN - would grow (committed 64 MB, usage +97%...)") and act never - the arena behaves exactly like a fixed v2 one +while you watch what the policy thinks. + +Observability: hg_stats (MI) shows committed/cap/grow_headroom, the +grows/grow_bytes/grow_refused/shrinks/shrink_bytes ledgers, the +grow_blocked gauge and the per-tier byte split; the same numbers export +as hgmem: statistics (hg_shm_committed, hg_shm_cap, hg_shm_grows, +hg_shm_grow_refused, hg_shm_grow_blocked, hg_shm_shrinks, ...). A +RESOURCE refusal that survives reclaim latches GROW-BLOCKED: one WARN, +the gauge goes 1, and E_CORE_SHM_GROW_BLOCKED is raised from the sweep +timer (subscribe with event_route; params arena/committed_mb/cap_mb/ +grow_refused) and re-raised every 5 minutes while latched. An admin +ceiling refusing is a NOTICE and never latches - a limit doing its job +is not an incident. + +Sizing rules that are not obvious: + + * tier 1 (hugetlb): the pool must fit the CAPS, not the initial + sizes - the whole -m cap is reserved from the pool at map time, and + every per-child pkg arena reserves its own -M cap the same way. On + top of that, leave free pages for the fork window: children briefly + COW-touch the parent's pkg arena before they build their own, each + touched page drawing a FREE page - a pool with zero free pages at + fork time SIGBUSes children (measured; it is the same crash family + as the 2026-08 pool-exhaustion incident). If the pool cannot fit a + cap, the arena falls back to a cap-less hugetlb reservation: it + keeps its huge pages and simply cannot grow, and says so. + * a 30-worker gateway with -M 16:64 exposes 30 x 64 MB of potential + pinned pkg growth. The RAM-floor check accounts for this, but the + admin should too. + * shrink on tier 1 returns pages to the POOL, not to general RAM - + freeing host memory needs the pool resized down as well. diff --git a/mem/README.hg_malloc_v3.md b/mem/README.hg_malloc_v3.md new file mode 100644 index 00000000000..6c22597b667 --- /dev/null +++ b/mem/README.hg_malloc_v3.md @@ -0,0 +1,1043 @@ +# HG_MALLOC v3 — the elastic arena + +HG_MALLOC v3 lets the shared-memory and per-process arenas **grow and +shrink at runtime**, between a starting size and a cap you reserve up +front. Undersizing no longer fails allocations at 3 a.m., and +oversizing no longer pins gigabytes of RAM around a workload that needs +megabytes — the arena follows the load, within limits you set, with +every decision observable and alertable. + +With no cap configured, **nothing changes**: the arena is fixed at +`-m`/`-M`, byte-for-byte the v2 behaviour. + +``` + hcap (the -m INIT:CAP reservation) + ┌────────────────────────────────────────────────┐ + │ committed (usable, pinned) │ reserved only │ + └─────────────────────────────┴──────────────────┘ + ↑ starts at INIT grows → ← shrinks + never below the shrink floor, never above the ceiling +``` + +## 0. Why an elastic arena + +Every OpenSIPS deployment ships with two numbers somebody guessed: +`-m`, the shared-memory arena all calls, dialogs and caches live in, +and `-M`, the private arena **each worker process** gets. Both are +fixed at boot. Both are a bet. + +**Bet low and you lose at the worst possible moment.** Shared memory +exhausts in the middle of your best traffic hour: allocations fail, +calls drop, and the only fix is a restart with a bigger number — +downtime, during the incident, on every node it touches. + +**Bet high and you pay for it every quiet hour.** The arena is pinned +physical memory; on the hugetlb tier it is carved out of the host at +boot and nothing else can ever use it. A production load balancer we +measured peaks at **11.9 MB** of shm over 15 hours of full traffic — +inside a 128 MB fixed arena. And `-M` multiplies: 16 MB across one +gateway's 54 workers is **864 MB** of committed RAM backing arenas that +mostly hold under 3 MB each, because with a fixed `-M` you size *every* +worker for the *busiest* worker's worst minute. + +And the bet cannot be won, because the right number is a moving +target — it shifts with traffic mix, dialog lifetimes, the module set, +the season: + +```mermaid +xychart-beta + title "One day, one node: fixed arena vs what the traffic needs (MB)" + x-axis ["00h","03h","06h","09h","12h","15h","18h","21h","24h"] + y-axis "MB" 0 --> 560 + line [512,512,512,512,512,512,512,512,512] + line [64,32,32,96,224,336,368,160,64] + line [41,17,15,83,201,318,344,138,49] +``` + +*Top, flat: a fixed `-m 512` sized for the storm, paid around the +clock. Middle, stepped: what v3 keeps committed — granule steps up +under load, shrink lagging the peak by the quiet window. Bottom: live +demand. (Illustrative shapes; a measured cycle is charted in +Section 8.)* + +v3 replaces both bets with a **range**: `-m 64:512 -M 16:32`. Start at +the size you can defend, reserve the ceiling (address space — free), +commit physical pages only as demand arrives, hand back what goes +quiet. Growth stops at three independent limits — your policy, the +backing tier, the host's real free RAM (pkg charged × the worker +count). Every decision is a counter you can scrape, the one state that +deserves a page is a latched event, and a dry-run mode narrates what it +*would* have done before you let it do anything. + +Every mechanism below was chosen **by measurement first** — including +two designs that looked right on paper and were proven memory-corrupting +before a line of allocator code was written. The log lines, MI output +and numbers in this document are real captures from the proof rigs, not +mockups. + +**Contents** + +0. [Why an elastic arena](#0-why-an-elastic-arena) +1. [Quickstart](#1-quickstart) +2. [Concepts and architecture](#2-concepts-and-architecture) +3. [Why the cap is on the command line](#3-why-the-cap-is-on-the-command-line) +4. [The memory tiers](#4-the-memory-tiers) +5. [Growth](#5-growth) +6. [The three-limit ceiling](#6-the-three-limit-ceiling) +7. [GROW-BLOCKED — the alertable state](#7-grow-blocked--the-alertable-state) +8. [Shrink](#8-shrink) +9. [The profile — configuration reference](#9-the-profile--configuration-reference) +10. [Dry-run mode](#10-dry-run-mode) +11. [pkg arenas — what is different](#11-pkg-arenas--what-is-different) +12. [Deployment cookbook](#12-deployment-cookbook) +13. [Monitoring and alerting](#13-monitoring-and-alerting) +14. [Log line reference](#14-log-line-reference) +15. [Sizing rules that are not obvious](#15-sizing-rules-that-are-not-obvious) +16. [Troubleshooting](#16-troubleshooting) +17. [Testing — the rig and how to reproduce the proofs](#17-testing--the-rig-and-how-to-reproduce-the-proofs) +18. [Appendix A — measured kernel facts](#18-appendix-a--measured-kernel-facts) +19. [Appendix B — internals map for developers](#19-appendix-b--internals-map-for-developers) +20. [Limitations](#20-limitations) + +--- + +## 1. Quickstart + +```bash +# 128 MB now, allowed to grow to 1 GB: +opensips -f opensips.cfg -m 128:1024 -M 16:64 -a HG_MALLOC +``` + +That alone gives you **exhaustion-triggered growth**: an allocation that +would have failed instead commits one 16 MB granule more and retries, +up to the cap. Loudly: + +``` +NOTICE:core:hg_malloc_init: shm HG_MALLOC_V3 arena: 128 MB on MAP_HUGETLB 2M pages, 128 MB pinned from swapping +NOTICE:core:hg_malloc_init: shm arena can grow to 1024 MB (896 MB headroom reserved, uncommitted) +... +NOTICE:core:hg_buddy_grow: shm arena grew by 16 MB to 144 MB (8 new pages on MAP_HUGETLB 2M pages; 880 MB headroom left) +``` + +Add a policy and it also grows **before** anything fails, and gives +quiet memory back: + +``` +auto_scaling_profile = MEM_SHM + scale up to 1024 on 80% for 3 cycles within 10 + scale down to 256 on 30% for 120 cycles + +auto_scaling_profile = MEM_PKG + scale up to 64 on 80% for 2 cycles within 5 + scale down to 8 on 20% for 60 cycles + +shm_auto_scaling_profile = MEM_SHM +pkg_auto_scaling_profile = MEM_PKG +``` + +The pkg profile governs **every worker's private arena individually**, +so its numbers are per-worker scale — an order of magnitude below the +shared arena's (Section 11). + +Watch it work: + +```bash +opensips-cli -x mi core:hg_stats # committed / cap / grows / shrinks / tiers +``` + +--- + +## 2. Concepts and architecture + +### 2.1 Committed vs reserved + +The arena block (`struct hg_block`) carries two sizes: + +| field | meaning | +|---|---| +| `hsize` | **committed** — pre-faulted, pinned, published to the buddy allocator, usable | +| `hcap` | **reserved** — the size of the one mapping created at startup | + +`hcap == hsize` (no `:CAP` given) is a fixed arena — exactly v2. + +### 2.2 The one invariant everything rests on + +**The whole cap is mapped once, `MAP_SHARED`, before fork.** Growth and +shrink never create, destroy, or re-protect mappings — they only change +which parts of that one shmem object are populated. + +This is forced, not stylistic. `mmap()` and `mprotect()` edit **one +process's page tables**, and the shm arena is shared by ~30 workers +that forked before any growth happens. The "obvious" design — keep the +tail `PROT_NONE` and `MAP_FIXED` deltas in on demand — was implemented +as a userspace rig first and **measured to be memory-unsafe in both +directions** on the fleet's oldest kernel (5.4): + +* **Growth** via post-fork `MAP_FIXED`: the grower reads its new pages + fine; a worker that forked earlier still has `PROT_NONE` there and + **SIGSEGVs** on the first cell handed out of grown space. (Control + arm: the pre-fork committed prefix was visible in the same worker.) +* **Shrink** via `mmap(PROT_NONE|MAP_FIXED)`: it rebinds only the + caller's mapping. Measured: after the shrinker released and re-grew a + range and wrote `0x77`, another worker **still read the old `0xEE`** + — two processes silently disagreeing about one arena address. + +With the whole-cap mapping, growth is a *commit* (populate + pin) and +shrink is a *punch* (`madvise`) on the shared object — both visible to +every process by construction. The untouched reserved tail costs only +page-table entries: a 64 MB mapped-untouched span was measured at +**576 kB of RSS**. + +### 2.3 The buddy grid grows without moving metadata + +The buddy allocator's per-page descriptors (`struct hg_page`, +leaf-order arrays, bitmaps) are laid out at init **for the full cap** +(`npages_cap`), not just the committed pages. The overhead is ~0.05% of +each never-committed page, paid up front — and it means a grow only +*publishes* pages that already have descriptors. The alternative would +be finding room for metadata in an arena that is, by definition of why +it is growing, full. + +Each page descriptor also carries the **achieved backing tier of the +commit that brought it in** (one byte, fits existing padding) — that is +what keeps the per-tier accounting truthful in both directions, since +shrink releases top pages that may come from any delta. + +### 2.4 The ownership registry records the cap + +`hg_arena_reg[]` (the cross-arena pointer-ownership table used on free +paths) records `hcap`, not `hsize`. Growth must never invalidate a +registry entry, or a pointer into freshly grown space would be misread +as foreign and "routed" to another arena. The whole cap's address range +belongs to this arena from reserve time; uncommitted ranges cannot hold +live cells, so the wider range cannot misattribute anything that +exists. + +--- + +## 3. Why the cap is on the command line + +The shm arena is created **before the config file is parsed** +(`init_shm_mallocs()` runs before `parse_opensips_cfg()` in `main()`), +and on the hugetlb tier the whole cap is reserved from the kernel's +page pool **at `mmap()` time** (measured — see Appendix A). There is no +later moment at which a config value could still shape the reservation. + +So the reservation lives where sizing always has: + +``` +-m INIT[:CAP] shared memory, MB +-M INIT[:CAP] per-process private memory, MB +``` + +`CAP` is rounded up to whole huge pages and must be ≥ `INIT` (refused +otherwise, at option parsing). Omitting `:CAP` pins the arena at +`INIT`. + +The config then supplies **policy within the reservation** — resolved +and validated in `init_shm_post_yyparse()`, the same post-parse hook +HP_MALLOC uses for memory warming. A profile can never raise the cap; +it can only choose how the space inside it is used. + +--- + +## 4. The memory tiers + +The arena tries four backings in order, each **attempted and then +verified through `/proc`** — never inferred from kernel version or +sysfs configuration: + +| tier | mechanism | verification | properties | +|---|---|---|---| +| 1 | `mmap(MAP_HUGETLB)` | mmap success | unswappable by construction; pool-accounted | +| 2 | `MADV_HUGEPAGE` before first touch | `/proc/self/smaps` PMD-mapped | THP at fault | +| 3 | `MADV_COLLAPSE` after fill | `ShmemHugePages` delta | THP retrofitted | +| 4 | plain 4 K | — | always works; mlock-pinned | + +Two v3-specific rules: + +* **Every growth delta re-negotiates its own backing.** A THP arena's + delta may land on 4 K next to memory that got 2 M at init. Backing is + an *outcome per range*, never an attribute of the arena — which is + why `hg_stats` reports a `tier_bytes` split whenever more than one + tier holds bytes, instead of one label that would be a lie. +* **Tier 1 reserves the whole cap from the pool at map time** + (measured: mapping 64 MB moved `HugePages_Rsvd` by exactly 32 pages + before any fault). Consequences: tier-1 growth can never fail + mid-flight — the pages are earmarked — and the pool must be sized for + the **caps** (Section 15). If the pool cannot fit the cap, the arena + falls back to a cap-less hugetlb reservation: it keeps huge pages, + cannot grow, and says so. + +Measured cost context (same workload, `perf` self-time across allocator +symbols): tier 1 ≈ 2.78%, tier 2 ≈ 2.97%, tier 3 ≈ 3.07%, tier 4 ≈ +3.38% — against F_MALLOC 7.56%. Huge pages are the smaller half of the +win; losing a delta to 4 K is a percent, not a disaster. + +--- + +## 5. Growth + +Two triggers, one mechanism. + +**Exhaustion (always armed once a cap exists).** The three points where +an allocation can die of buddy exhaustion — small-object chunk carving, +the large tier, region allocation — each grow-and-retry exactly once. +If the arena grew, the freshly published whole pages satisfy the retry +by construction; a second miss can only mean the grow itself was +refused, and the caller's ordinary exhaustion error follows. + +**Proactive (with a profile).** Once per sweep interval (30 s), usage +(carved-of-committed) is compared to the profile's up-threshold using a +cycles-in-window count. Crossing it grows one granule **before any +allocation fails**. The exhaustion path stays armed for bursts between +ticks. + +The commit itself (`hg_mem_commit()`): + +1. optional host-RAM check (Section 6) — before any work; +2. `mlock()` of the delta — chosen because it populates *exactly* that + range, pins it, and reports failure through `errno` instead of + letting a worker SIGBUS later on half-committed memory. Growth is + **refused, never degraded**; +3. per-delta tier verification (Section 4), `tier_bytes` accounting; +4. only then: `hsize` moves, pages are published, the reserve floor is + recomputed, counters tick. + +The pre-fault runs under the arena lock — a deliberate, bounded trade. +Growth is once per granule of genuine demand; every other worker in the +slow path at that moment is *also* out of memory and would only queue +on the same growth. + +Granule: 16 MB (huge-page rounded). + +--- + +## 6. The three-limit ceiling + +`min(admin, tier, host RAM)` — each limb enforced where it is real: + +| limb | what | where enforced | +|---|---|---| +| admin | the profile's scale-up target, else the `-m` cap | `hg_buddy_grow()` — refusing it is a NOTICE, never an alert: a limit doing its job is not an incident | +| tier | hugetlb pool | **at reserve time** (map-time pool reservation — measured); tiers 2–4 additionally by `mlock`'s own errno | +| host RAM | `MemAvailable` vs a floor | `hg_grow_ram_refused()` before each commit | + +The RAM floor defaults to `max(256 MB, MemTotal/20)`, configurable: + +``` +hg_ram_floor_mb = 2048 +``` + +Tier 1 **skips** the RAM limb outright: its pages were carved out of +host RAM when the pool was created — charging them again would +double-count (also measured, not assumed). + +**pkg deltas are charged × the process count**: every worker grows its +own private arena under the same workload, so the single-arena delta +understates the real host cost ~30× on a gateway. Verified +differentially — with one floor, a 16 MB pkg delta was refused while +16 MB shm deltas grew: + +``` +WARNING:core:hg_grow_ram_refused: pkg: refusing to grow by 16 MB: 144 MB effective (x9 processes) would leave the host under the 14589 MB floor (MemAvailable 14562 MB). Freeing host memory or lowering the floor lifts this. +``` + +The whole grow path, trigger to publish: + +```mermaid +flowchart TD + A["allocation would fail + (chunk carve / large tier / region)"] -- "grow-and-retry" --> G + P["proactive tick: + profile up-window met"] --> G + G["grow request, whole 16 MB granules"] --> L1{"admin ceiling: + profile up-target, else the cap"} + L1 -- "at ceiling: NOTICE, refused++" --> R["refusal counters + (hg_shm_grow_refused / pkg twin)"] + L1 -- ok --> L2{"tier: does the backing + (hugetlb pool) cover the delta?"} + L2 -- no --> R + L2 -- ok --> L3{"host RAM floor: + MemAvailable - delta >= floor + (pkg: delta x worker count)"} + L3 -- under --> R + L3 -- ok --> C["mlock() the new granules: + commit, verify tier per delta"] + C --> W["publish pages to the buddy grid + grows++, headroom NOTICE, cooldown armed"] + R -- "resource refusals only" --> B["GROW-BLOCKED machinery + (Section 7)"] +``` + +--- + +## 7. GROW-BLOCKED — the alertable state + +A **resource** refusal (host RAM, mlock limit — not an admin ceiling) +should page someone *only if it means something*. The state machine: + +```mermaid +stateDiagram-v2 + [*] --> idle + idle --> armed: resource refusal + armed --> idle: quiet for a full sweep interval + armed --> LATCHED: refusal recurs after a GC pass + armed --> LATCHED: full sweep interval, refusals still accruing + LATCHED --> idle: a grow succeeds + LATCHED --> idle: demand falls below the floor recovery mark + note right of LATCHED + gauge hg_shm_grow_blocked = 1 + one WARN + E_CORE_SHM_GROW_BLOCKED + re-raised every 5 min while held + end note +``` + +Two latch routes exist because of a measurement: the original +"survived a GC pass" rule alone sat through **5 million refusals with +`gc_passes == 0`** — a full arena where nothing is reclaimable is +exactly the state that most needs the alert, and it never runs GC. The +sweep timer is therefore a second promoter: armed + still refusing a +full interval later ⇒ latch. + +The hysteresis was verified in both directions: isolated +one-refusal-per-interval spikes never latch (each arming is disarmed by +the next quiet tick); a sustained stream latches. + +Surfaces: the `hg_shm_grow_blocked` **gauge** (the alertable one), one +WARN log line, and the `E_CORE_SHM_GROW_BLOCKED` event — raised **from +the sweep timer**, never under the arena lock: `evi_raise_event()` +allocates shm, and raising it inside the allocator that just refused to +grow would re-enter a full arena. + +``` +event_route[E_CORE_SHM_GROW_BLOCKED] { + xlog("L_WARN", "arena $param(arena) blocked: committed $param(committed_mb)MB / cap $param(cap_mb)MB, $param(grow_refused) refusals\n"); +} +``` + +The event is published with the other core events unconditionally, so +`event_route` can always subscribe; only the *raise* is conditional. +Shm arena only, honestly so: each pkg arena's state is private to its +process (its WARN appears in that worker's log, its counters in its own +`hg_stats` pkg section). + +Log flood protection is separate from the latch and applies everywhere: +refusal details are logged **once per episode** (re-armed by the next +successful grow). The first at-cap soak without this printed 239,458 +identical NOTICEs in four seconds; the counter carries the magnitude, +the log carries the fact. + +--- + +## 8. Shrink + +### 8.1 Why it is safe with zero cross-process coordination + +Only pages the buddy proves **wholly free** can be released, and only +from the **top** of the committed range: + +* *Whole-free is a proof, not a heuristic.* Buddy merging is eager, so + an all-leaves-free page has provably merged into one top-order block. + And a cell parked in any thread's private `__thread` cache has **not** + decremented its block's live count — that happens only at cache/block + transitions — so its block is still carved and its page can never + appear whole-free. Contrapositive: whole-free ⇒ no live pointer into + the page exists anywhere ⇒ nothing to coordinate. +* *Top-only* keeps `hg_owns()` one contiguous range test and the + ownership registry valid — the address-space invariants of the whole + allocator. + +### 8.2 The primitive — measured, with the wrong answers named + +`munlock()` + `madvise(MADV_REMOVE)` for the shared arena: it punches +the **shmem object**, so every mapped process is affected. Measured on +kernel 5.4: + +* frees the pages even while another process holds them `VM_LOCKED` + (worker RSS dropped by exactly the punched size; re-read returned + zeroes); +* the range recommits cleanly afterwards (mlock + write, visible + cross-process); +* on hugetlb, the pages **return to `HugePages_Free`** and are drawn + back out on re-fault — the pool round-trip. + +pkg arenas (`MAP_PRIVATE`) use `MADV_DONTNEED` — per-process memory, +no cross-process question exists. + +The one primitive that must never be used is +`mmap(PROT_NONE|MAP_FIXED)` over the range — see 2.2; it was measured +leaving other workers reading stale bytes. + +A kernel that refuses the advice latches `shrink_unsupported` once and +the arena simply stays grown — nothing retries, nothing spams. + +### 8.3 Ordering and policy + +The punch runs **before** any bookkeeping, under the arena lock, so a +refused release changes nothing and no worker can carve from pages +mid-punch. On success: pages leave the free lists and the grid, +`hsize`/`npages`/floor/counters adjust, per-page tiers decrement +`tier_bytes`. + +Cadence is down-slow by design: one granule per quiet window +(no-profile default: 4 consecutive quiet sweep intervals = 2 minutes; +with a profile: its own threshold and cycle count). Hard safety +conditions hold regardless of policy — never below the floor, never +while grow-blocked, top page must already be whole-free, and free space +must stay clear of the reserve floor's recovery threshold even after +giving the granule back, so a shrink can never re-trigger the pressure +that would regrow it. Any grow resets the window and starts the +profile's post-grow cool-off (10× the down-cycles). + +**Prefer-low allocation** makes tops drain: the top-order free list is +kept in ascending address order, so whole-page carves always take the +lowest free page; multi-page runs already scanned ascending. Lower +orders stay LIFO — their placement belongs to the cell-level +concentration policy, and their lists are the long ones where an +ordered walk would cost. + +Floor: never below `-m`/`-M`'s initial size — unless a profile says +otherwise (Section 9): with a profile attached, the profile is the ask +and `-m` is just the starting size. + +A full cycle, measured on a live process (rig arm K, Section 17): grow +under MI-driven load, cooldown, the shrink train once the quiet window +passes, regrowth on the next pulse — committed walking 32→96→16→80 MB +while the reservation never moves and the hugetlb pool gets every +released page back: + +```mermaid +xychart-beta + title "Measured: committed MB through one load cycle (reservation fixed)" + x-axis ["idle", "hold", "cooldown", "quiet + shrink train", "second hold"] + y-axis "committed MB" 0 --> 110 + bar [32, 96, 96, 16, 80] + line [32, 96, 96, 16, 80] +``` + +--- + +## 9. The profile — configuration reference + +v3 reuses the worker autoscaler's grammar — same tokens, same shape +your configs already use for `use_auto_scaling_profile`: + +``` +auto_scaling_profile = + scale up to on % for cycles [within ] + [scale down to on % for cycles] + +shm_auto_scaling_profile = +pkg_auto_scaling_profile = # optional, may be a different profile +hg_ram_floor_mb = # 0 = auto (max(256MB, MemTotal/20)) +hg_autoscale_dry_run = 0|1 +``` + +| element | meaning for an arena | +|---|---| +| `up to N` | growth **ceiling**, MB, huge-page rounded — the admin limb, within the `-m` cap | +| `on P% for C within W` | grow when usage ≥ P% in C of the last W cycles (`within W` omitted ⇒ W = C) | +| `down to M` | shrink **floor**, MB — may be *below* `-m` | +| `on Q% for C` | shrink one granule after C consecutive cycles at ≤ Q% | +| one cycle | one sweep interval (30 s) | +| implicit | post-grow cool-off of 10×C cycles before shrink counting resumes | + +Usage is carved-of-committed. Profile numbers are copied **into** the +(possibly shared) arena block at attach — never pointed to; the profile +structs live in process-local memory. + +### Validation — fail-loud, all real messages + +A profile that cannot work stops startup with the reason: + +``` +ERROR: shm_auto_scaling_profile 'MEM_SHM' does not name an auto_scaling_profile +ERROR: shm profile 'MEM_SHM': the arena has no growth room - give the reservation on the command line (-m INIT:CAP) +ERROR: shm profile 'MEM_SHM': scale-up target 96 MB does not exceed the initial 128 MB - the profile could never act +ERROR: shm profile 'MEM_SHM': scale-up target 2048 MB exceeds the 1024 MB reservation - raise the :CAP +ERROR: shm profile 'MEM_SHM': scale-down target 2 MB is below the 4 MB minimum viable arena +ERROR: shm profile 'MEM_SHM': scale-down target 512 MB is not below the scale-up target 256 MB +``` + +A profile named while a different allocator runs is ignored with a +WARN. Every accepted profile announces itself once, so a config's +effect is verifiable from the log alone: + +``` +NOTICE:core:hg_autoscale_apply: shm auto-scaling profile 'MEM_SHM': 32..160 MB (start 64), up at 60% for 2/3 cycles, down at 20% for 1 cycles (cooldown 10) +``` + +--- + +## 10. Dry-run mode + +``` +hg_autoscale_dry_run = 1 +``` + +Advise-only: ticks **and even emergency exhaustion growth** log what +they would have done and act never — the arena behaves exactly like a +fixed v2 arena while you watch what the policy thinks: + +``` +NOTICE:core:hg_grow_tick: shm: DRY RUN - would grow (committed 64 MB, usage 97%, profile ceiling 160 MB) +WARNING:core:hg_buddy_grow: shm: DRY RUN - would grow for a 285488 byte request (committed 64 MB); counting further suppressed grows in hg_shm_grow_refused +NOTICE:core:hg_shrink_tick: shm: DRY RUN - would shrink (committed 96 MB, usage 12%) +``` + +Suppressed grows still count in `grow_refused`, so the *volume* of what +dry-run declined is measurable, not just its existence. Proven: a +dry-run arena at 97% usage under held load produced the advice lines +and **zero** actual grows, committed unchanged. + +Deployment pattern: ship the profile with dry-run on, watch a few days, +tune thresholds against the advice lines, flip to 0. + +--- + +## 11. pkg arenas — what is different + +* **Per-child, post-config.** Each worker's private arena is created at + fork time (after the config is parsed), so it takes the full policy: + cap from `-M INIT:CAP`, profile numbers from a fork-inherited + resolution. +* **The pre-fork parent arena stays fixed** at `-M`'s initial size — it + predates the config, and the attendant barely allocates. Documented + cost: none in practice. +* **Ticks run in the owner.** Only a process can shrink its own private + arena; the pkg grow/shrink gates tick from each process's sweep-flush + path, once per interval, same cadence as shm. +* **Host costs multiply.** Both the RAM-floor check (automatic) and + your capacity plan (manual) must charge pkg deltas × the worker + count. `-M 16:64` on a 30-worker gateway is up to 1.9 GB of potential + pinned growth — and on tier 1, the same multiplication applies to + **pool reservations** (each child's arena reserves its own cap at map + time). + +What the multiplier means on a real 54-worker gateway — to survive one +worker's burst with a fixed `-M` you must hand the burst size to all +54; with a range, the floor is the norm and the burst is one worker's +temporary excursion: + +```mermaid +xychart-beta + title "pkg, 12 of 54 workers: fixed -M sized for the burst vs v3 (MB)" + x-axis ["w1","w2","w3","w4","w5","w6","w7","w8","w9","w10","w11","w12"] + y-axis "MB per worker" 0 --> 36 + bar [32,32,32,32,32,32,32,32,32,32,32,32] + bar [16,16,16,32,16,16,16,16,16,16,16,16] +``` + +*First bars: fixed `-M 32` because worker 4 once needed 32 — +1,728 MB across 54 workers, around the clock. Second bars: v3 with +`-M 16:32` — 53 workers hold the 16 MB floor, the burst worker grows a +granule and returns it after the quiet window; peak 880 MB. When the +profile's down-target sits below `-M`, the floor drops further still. +(Illustrative; the five-worker independence proof is below.)* + +Proven: five workers each grew their own arena 8→24→40 MB +independently, on per-delta-verified THP backing, with every stamped +word intact. + +--- + +## 12. Deployment cookbook + +### 12.1 Billing gateway, hugetlb tier + +Start at the proven working size, allow storm growth, never shrink +below the start (a gateway holds long-lived state). + +```bash +# /etc/default/opensips +MALLOC=HG_MALLOC +SHM_MEMORY=128:1024 +PKG_MEMORY=16:32 +``` + +``` +# opensips.cfg +auto_scaling_profile = MEM_SHM + scale up to 1024 on 75% for 2 cycles within 5 + scale down to 128 on 25% for 120 cycles + +shm_auto_scaling_profile = MEM_SHM +``` + +```bash +# /etc/sysctl.d/60-opensips.conf — the pool must fit the CAPS: +# shm cap: 1024 MB = 512 pages +# pkg cap × 31 workers: 32 MB × 31 = 496 pages +# rounding / restart headroom (~5%) ≈ 50 pages +vm.nr_hugepages=1060 +``` + +```ini +# systemd drop-in +[Service] +LimitMEMLOCK=infinity +``` + +Only the caps draw from the pool. The pre-fork (attendant) pkg arena — +the one every child inherits copy-on-write — is deliberately **not** +hugetlb-backed: a child's COW fault inside a hugetlb mapping has no 4K +fallback and no reservation, so an empty pool at fork time was a silent +SIGBUS (measured; it is how TCP main, the last no-script child, died at +startup on a short pool — twice). Its ladder starts at THP instead, +whose COW splits to 4K pages, and no-script children no longer walk the +inherited route AST with `pkg_free` before swapping to their own arena. +Startup therefore cannot SIGBUS on pool state: a short pool only pushes +late children down the tier ladder. If the pool cannot fit a cap: + +``` +NOTICE: hugetlb pool cannot back a 1024 MB cap; reserving the 128 MB in use instead - the arena keeps huge pages but cannot grow. Raise vm.nr_hugepages to allow growth. +``` + +### 12.2 Load balancer, measured-first + +An LB's real footprint is small (a production LB measured **11.9 MB +peak shm over 15 h** of full traffic; worst per-process pkg 2.4 MB). +Start small, keep storm headroom, hand idle memory back: + +```bash +opensips -f lb.cfg -m 64:512 -M 16 -a HG_MALLOC +``` + +``` +auto_scaling_profile = MEM_LB + scale up to 512 on 70% for 2 cycles within 4 + scale down to 32 on 20% for 20 cycles + +shm_auto_scaling_profile = MEM_LB +``` + +`down to 32` sits below `-m 64` deliberately: after a storm passes, the +arena returns even part of the initial allocation. + +### 12.3 First rollout — dry-run + +Section 10. Profile + `hg_autoscale_dry_run = 1`, observe, tune, flip. + +### 12.4 Cap only, no profile + +```bash +opensips -f opensips.cfg -m 256:2048 -a HG_MALLOC +``` + +Exhaustion growth + conservative built-in shrink (4 quiet intervals per +granule, never below `-m 256`). No proactive behaviour, no config +surface at all. + +### 12.5 The first soak, charted + +Two nodes, first 14 hours on v3 (2026-08-14/15), every point taken from +the arena's own grow/shrink NOTICE lines. The LB ran the section 12.2 +recipe untouched; the gateway was deliberately re-cut mid-soak to a +near-empty start (`-m 8:512 -M 2:32`) to make growth earn everything. + +```mermaid +xychart-beta + title "LB .250: committed MB (top line shm, flat line pkg)" + x-axis ["23:49 boot", "23:59", "00:09", "08:19", "10:09", "14:00"] + y-axis "committed MB" 0 --> 70 + line [64, 48, 32, 48, 34, 34] + line [16, 16, 16, 16, 16, 16] +``` + +*The 12.2 config doing its job unattended: idle 64 shrinks to the 32 +floor within 20 minutes of boot, morning traffic grows it back to 48, +and the after-peak shrink releases only what is genuinely empty — +committed lands on 34, not 32, because 2 MB of the growth still holds a +live allocation ("7 pages released; 2 MB of growth still held"). pkg +never moved on any of the 28 workers. Changes are instantaneous steps; +the slopes are an artifact of the event-spaced axis.* + +```mermaid +xychart-beta + title "GW .244: committed MB (shm; pkg typical; pkg 3 grown workers)" + x-axis ["23:50 boot", "00:10", "00:30", "00:57 restart", "01:17 re-cut", "01:19", "14:00"] + y-axis "committed MB" 0 --> 70 + line [64, 48, 32, 64, 8, 24, 24] + line [16, 16, 16, 16, 2, 2, 2] + line [16, 16, 16, 16, 2, 18, 18] +``` + +*The adversarial arm. First boot (`-m 64`) runs the predicted shrink +train to 32; a restart resets to 64; then the re-cut starts shm at +**8 MB** and per-worker pkg at **2 MB**. Demand pulls shm to 24 within +two minutes and exactly three of 54 workers grow their pkg to 18 — +the other 51 stay at the 2 MB floor. Twelve hours of steady state +followed.* + +Both nodes, the whole window: `grow_refused` 0, `grow_blocked` 0, +corruption counters 0, tier-1 hugetlb throughout, and the +`E_CORE_SHM_GROW_BLOCKED` event never fired. + +--- + +## 13. Monitoring and alerting + +### 13.1 MI + +```bash +opensips-cli -x mi core:hg_stats +``` + +```json +"shm": { + "tier": "plain 4K pages", + "committed": 167772160, + "cap": 268435456, + "grow_headroom": 100663296, + "grows": 6, + "grow_bytes": 100663296, + "grow_refused": 1, + "grow_blocked": 0, + "shrinks": 8, + "shrink_bytes": 134217728, + ... +} +``` + +Field notes: + +| field | meaning | +|---|---| +| `tier` | what **init** achieved | +| `tier_bytes` | per-tier byte split — appears only when >1 tier holds bytes; the honest answer to "is my grown arena still on huge pages" | +| `committed` / `cap` / `grow_headroom` | the elastic state; `committed == cap` ⇒ fixed or fully grown | +| `grows` / `grow_bytes` | successful commits and their total | +| `grow_refused` | refusals, both admin and resource — the magnitude counter behind once-per-episode logging | +| `grow_blocked` | the latched gauge (resource refusals only) | +| `shrinks` / `shrink_bytes` | successful releases and their total | + +The `pkg` section reports the **answering MI process's own** arena — +stated honestly rather than pretending fleet-wide pkg visibility. + +### 13.2 Statistics (Prometheus-friendly) + +``` +hgmem:hg_shm_committed bytes committed right now +hgmem:hg_shm_cap the reservation +hgmem:hg_shm_grows counter +hgmem:hg_shm_grow_bytes counter +hgmem:hg_shm_grow_refused counter — rising = demand is hitting a wall +hgmem:hg_shm_grow_blocked GAUGE — the one to alert on +hgmem:hg_shm_shrinks counter +hgmem:hg_shm_shrink_bytes counter +``` + +Suggested alerts: + +* `hg_shm_grow_blocked == 1` for > 1 min → page. This is "demand + present, host cannot supply, reclaim did not help". +* `rate(hg_shm_grow_refused[10m]) > 0` with `grow_blocked == 0` → + ticket, not page: the admin ceiling is being hit, or refusals are + isolated; check whether the ceiling still matches reality. +* `hg_shm_committed / hg_shm_cap > 0.9` sustained → the cap is close; + plan a restart with a larger `:CAP` (the reservation cannot be raised + live). + +### 13.3 The event + +Section 7 — `E_CORE_SHM_GROW_BLOCKED`, params `arena`, `committed_mb`, +`cap_mb`, `grow_refused`; raised on latch and every 5 minutes while +held; with no subscriber, a WARN says so and points at the gauge. + +--- + +## 14. Log line reference + +All at their exact severities; `%` values are illustrative. + +| line | severity | meaning / action | +|---|---|---| +| `shm arena can grow to N MB (M MB headroom reserved, uncommitted)` | NOTICE | startup: a cap exists | +| `shm auto-scaling profile 'X'...: A..B MB (start C), up at ...` | NOTICE | profile attached; the one line that proves your config took effect | +| `... [DRY RUN - advise only]: ...` | NOTICE | ditto, advise-only | +| `shm arena grew by 16 MB to N MB (8 new pages on ; M MB headroom left)` | NOTICE | growth, with the delta's **verified** backing | +| `shm arena shrank by 16 MB to N MB (8 pages released to the ; M MB of growth still held)` | NOTICE | shrink, with where the memory went | +| `at the N MB growth ceiling (the -m/-M reservation \| the profile scale-up target), a K byte request must fail - counting further refusals in hg_shm_grow_refused` | NOTICE | admin limb refusing; once per episode; not an incident | +| `cannot grow by 16 MB: mlock failed (...)` / `refusing to grow by 16 MB: N MB effective (xP processes) would leave the host under the F MB floor` | WARN | resource limb refusing; once per episode | +| `GROW-BLOCKED latched - the arena cannot grow and a did not change that (N refusals so far)` | WARN | the latch; gauge is now 1 | +| `GROW-BLOCKED cleared - ` | NOTICE | recovery | +| `DRY RUN - would (...)` | NOTICE/WARN | advise-only decisions | +| `hugetlb pool cannot back a N MB cap; reserving the M MB in use instead` | NOTICE | pool < cap at startup; arena fixed on huge pages | +| `mlock of the N MB HG_MALLOC arena failed (...): continuing unpinned` | WARN | init pin failed (RLIMIT); non-fatal, arena swappable | +| `cannot release memory (MADV_... failed): shrink disabled for this arena` | WARN | kernel refused the primitive; once, permanent for the run | + +--- + +## 15. Sizing rules that are not obvious + +1. **Tier 1: the pool must fit the caps.** The whole `-m` cap is + reserved from the hugetlb pool at map time, and every per-child pkg + arena reserves its own `-M` cap the same way. Budget + `shm_cap + pkg_cap × workers + ~12% margin` pages. +2. **A short pool degrades, it no longer kills.** The one arena + children inherit copy-on-write — the attendant's pkg arena — is kept + off hugetlb (12.1), so a pool with zero free pages at fork time + pushes late children to THP instead of SIGBUSing them. That SIGBUS + was measured, twice, before this rule existed. +3. **pkg caps multiply** — RAM and, on tier 1, pool reservations. +4. **Tier-1 shrink returns pages to the pool, not to RAM.** Freeing + host memory requires shrinking `vm.nr_hugepages` as well. +5. **When verifying pool behaviour, never read `HugePages_Free` + alone.** Per-process *private* hugetlb (each worker's pkg arena and + its copy-on-write touches of the inherited parent arena) moves the + same counter, in the opposite direction, at a similar pace. Compute + + ``` + object_pages = (HugePages_Total − HugePages_Free) − Σ Private_Hugetlb(all pids) + ``` + + from `/proc/meminfo` + `/proc//smaps`. Three consecutive test + rigs misread a *working* tier-1 shrink as broken before this was + applied; the exact measurement showed the shared object stepping + 96 → 16 MB precisely as `committed` claimed (the 3-page remainder + was the `shm_dbg` pool). +6. **A small negative `HugePages_Rsvd` (reads as huge unsigned) can + appear** on nodes running per-child pkg arenas — a kernel accounting + drift tied to the abandoned-parent-arena COW pattern, bounded to a + few pages of overstated headroom. Known, monitored, not caused by + v3. + +--- + +## 16. Troubleshooting + +| symptom | cause | fix | +|---|---|---| +| startup: `the arena has no growth room` | profile attached but no `:CAP` on `-m`/`-M` | add the cap: `-m 128:1024` | +| startup: `scale-up target ... exceeds the ... reservation` | profile wants more than the cap | raise `:CAP` or lower the target | +| startup: `does not name an auto_scaling_profile` | typo, or the profile block is below the `shm_auto_scaling_profile` line in a way the parser never saw | check `auto_scaling_profile = NAME` exists and parses | +| `hugetlb pool cannot back a ... cap` at init | `vm.nr_hugepages` smaller than the caps | grow the pool (rule 1), restart | +| arena grew but new pages are 4 K on a THP host | each delta negotiates independently | expected; see `tier_bytes`; consider tier 1 for guarantees | +| never shrinks | top page busy, or inside the post-grow cool-off (10× down-cycles), or usage above the down-threshold, or below-floor/blocked safety hold | check `hg_stats`; prefer-low needs time to drain the top | +| `DRY RUN - would grow` but nothing happens | `hg_autoscale_dry_run = 1` | that is the point; set 0 to act | +| `grow_refused` climbing, gauge 0 | isolated refusals; hysteresis holding | by design — the gauge latches on *sustained* refusal | +| `failed to initialize child process N` / `cannot fork tcp main` at startup, no arena line for that child | a build predating `HG_INIT_INHERITED`: the last no-script child COW-faulted the parent's hugetlb pkg arena on an empty pool | upgrade; meanwhile leave free pages in the pool at fork time | +| arena runs unpinned (`continuing unpinned`) | `RLIMIT_MEMLOCK` too low for a tier 2–4 arena | `LimitMEMLOCK=infinity` in the unit | +| testing under `ulimit -l` shows no refusals | you are root — `CAP_IPC_LOCK` bypasses `RLIMIT_MEMLOCK` entirely | test the mlock leg as an unprivileged user (`setpriv`) | +| pool numbers "prove" shrink is broken | rule 5 | use the exact object-residency formula | + +--- + +## 17. Testing — the rig and how to reproduce the proofs + +### 17.1 hgstress + +`modules/hgstress` is the throwaway stress module. Every block is +stamped per (pid, slot) in every 8-byte word, so a page served to two +processes, a lost write, or a punch that ate live data is caught and +named, not inferred. + +| param / MI | purpose | +|---|---| +| `slots`, `iters`, `verify`, `large` | the classic multi-process churn soak | +| `hold_mb` | each worker allocates and HOLDS N MB of stamped 128–512 K blocks through the churn — the growth driver; sized past `-m` it forces growth with every worker live | +| `hold_pkg_mb` | same driver for each worker's private arena | +| `again_s` | a timer re-runs one hold/verify/free cycle N seconds in — the regrow-after-shrink proof (timers only run after `child_init` completes, so this is how post-startup cycles are driven) | +| MI `hgs_hold ` / `hgs_release` | allocate/park and verify/free stamped shm **from a live MI process** — the only way to meet sweep ticks, since `child_init` soaks block every timer | + +### 17.2 What each proof arm established + +| arm | shape | result | +|---|---|---| +| A | fixed 64 MB, demand 120 MB | fail-first control: 109,582 refusals, 0 grows, 0 torn | +| B | cap 256 MB | 7 grows by four different worker pids, 0 refusals, 5/5 PASS — cells in pages committed post-fork verified by pre-fork processes | +| C | cap 96 < demand | grows to cap, ONE at-cap NOTICE (the 239 k-flood fix), refusals counted | +| D | pkg caps | five workers grew their own arenas independently on verified THP | +| E | hugetlb pool | growth on tier 1 to the cap, pool accounting exact; also the zero-margin SIGBUS reproduction | +| F/G | RLIMIT & RAM-floor | the resource limb: latch, gauge, live `event_route` delivery; the ×nproc differential | +| H | grow→free→quiet→regrow | the full ellipse incl. 12 shrinks and a timer-driven regrow INTO punched ranges, 0 torn | +| I | profile | attach line, proactive grow with zero exhaustion errors, ceiling exactly at the profile target, cool-off to the tick, shrink to a below-`-m` floor, dry-run advising with zero action | +| J | non-root, `ulimit -l` | init-unpinned path + the growth-mlock refusal root cannot drive + hysteresis in both directions | +| K | tier-1 lifecycle | pool draw / ceiling / **pool return** / re-draw — plus the instrumentation lesson of rule 5 | + +### 17.3 The userspace pre-measurement rigs + +`vatest.c`, `growtest.c`, `shrinktest*.c` (session scratchpad) are the +kernel-behaviour probes that chose the mechanisms before any allocator +code was written — each with a control arm that passes. Appendix A is +their output. + +--- + +## 18. Appendix A — measured kernel facts + +All on Linux 5.4 (the fleet's oldest) unless noted; every claim +re-verified rather than assumed from documentation. + +| # | fact | measurement | +|---|---|---| +| 1 | post-fork `MAP_FIXED` into a `PROT_NONE` reservation is invisible to pre-forked processes | control prefix readable; delta SIGSEGV in the sibling | +| 2 | a whole-cap pre-fork `MAP_SHARED` mapping makes later commits visible everywhere | delta written by parent read correctly by pre-forked child | +| 3 | an untouched mapped span is nearly free | 64 MB span: 576 kB RSS | +| 4 | `mlock(sub-range)` populates and pins exactly that range | RSS/VmLck moved by precisely the delta; tail untouched | +| 5 | `mmap(PROT_NONE\|MAP_FIXED)` "shrink" silently corrupts a shared arena | shrinker wrote 0x77; sibling still read 0xEE | +| 6 | `MADV_REMOVE` punches the object for every mapper | sibling read zeroes; recommit visible | +| 7 | `MADV_REMOVE` frees pages held `VM_LOCKED` by another process | locker's RSS fell by the punched size | +| 8 | hugetlb `MADV_REMOVE` works on 5.4 and returns pages to the pool | `HugePages_Free` 16→20 on an 8 MB punch; re-fault drew them back | +| 9 | `MAP_HUGETLB` reserves the whole mapping from the pool at map time | `Rsvd` +32 pages for a 64 MB map, before any fault | +| 10 | `MAP_NORESERVE` hugetlb takes nothing at map time (SIGBUS risk) | `Free` unmoved | +| 11 | a hugetlb pool with zero free pages SIGBUSes forking children (COW window) | reproduced; `si_addr` inside the parent's private pkg arena | +| 12 | punch behaviour is unchanged by `MADV_DONTDUMP`, many mappers, foreign mlock, or a forked puncher | isolation arms all returned pages | +| 13 | `HugePages_Free` alone cannot judge object residency | three rigs misread a working shrink; private fault-in (63→71 pages) masked the returns; exact formula in rule 5 | +| 14 | root cannot test `RLIMIT_MEMLOCK` | `CAP_IPC_LOCK` bypasses the limit entirely; `setpriv --reuid=nobody` + `ulimit -l` drives the leg | +| 15 | `mlock` on a reservation-backed hugetlb range cannot SIGBUS mid-commit | clean errno contract held through every arm | + +--- + +## 19. Appendix B — internals map for developers + +| file | owns | +|---|---| +| `mem/hg_malloc.c` | reservation (`hg_mem_reserve`), commit (`hg_mem_commit`), release (`hg_mem_release`), the tier ladder + verification probes, RAM limb (`hg_grow_ram_refused`), profile attach (`hg_autoscale_post_cfg`/`hg_autoscale_apply`), init/destroy | +| `mem/hg_malloc.h` | `struct hg_block` incl. `hcap`/`hsize_min`/policy copy/latch state/`tier_bytes[]`; the cap globals' contract | +| `mem/hg_buddy.c` | grow (`hg_buddy_grow`) + retry contract, shrink (`hg_buddy_shrink`), the policy ticks (`hg_grow_tick`/`hg_shrink_tick`), latch helpers (`grow_resource_refused`/`hg_grow_unblock`/`hg_grow_blocked_tick`), prefer-low `fl_push` | +| `mem/hg_arena.c` | the exhaustion call sites (carve/region), `npages_cap` layout, floor-recovery unblock hook, the statistics table | +| `mem/hg_large.c` | the large tier's grow-and-retry loop | +| `core_stats.c` | the sweep timer: cache sweep, deferred event raise, shm ticks | +| `main.c` / `globals.c` / `globals.h` | `-m INIT:CAP` parsing; the six always-present globals | +| `cfg.lex` / `cfg.y` | the four config tokens; the (pre-existing, reused) profile grammar | +| `evi/evi_core.[ch]` | `E_CORE_SHM_GROW_BLOCKED` publication (id 6; carries the same `#ifdef STATISTICS` id-shift caveat as `SHM_THRESHOLD`) | +| `mem/shm_mem.c` | the `init_shm_post_yyparse()` attach call | +| `modules/hgstress/` | the proof driver (17.1) | + +Development notes that cost real time, recorded so they are paid once: + +* gcc 9 (build host 222) does not flag a read-before-assign introduced + by restructuring a loop condition; the resulting garbage pointer + (`&evi_time_str+12`) survived one full arm before the rig's detector + caught it via a zeroed `log_level`. Build on the newer-gcc hosts for + warning coverage; keep the rigs' detectors on. +* `child_init` work blocks every timer until the last child finishes — + policy ticks and event raises cannot be observed from a `child_init` + soak; drive load via MI. +* opensips under `setpriv`/`sh` wrote nothing to stderr in the non-root + arm — judge such runs by MI counters, not logs. +* A version-stamp check (`version_control`) refuses mixed core/module + revisions after any commit — full-tree rebuilds between rig runs. + +--- + +## 20. Limitations + +* The **pre-fork parent** pkg arena predates the config and stays fixed + at `-M`'s initial size; profiles govern every per-child arena. +* Shrink is **top-only**; a single live cell in the top page blocks + release until it moves or dies. Prefer-low allocation drains tops + over time, but does not relocate live cells — relocation was + evaluated separately and is unsound here (raw pointers, interior + pointers, no safepoints), and address-ordered coalescing beyond the + page was **closed by measurement**: on four production node shapes, + free space already sat 97–99% in whole huge pages. +* The reservation (`:CAP`) cannot be raised live — it is a mapping (and + on tier 1 a pool reservation) created before fork. Raising it is a + restart. +* Growth granule is fixed at 16 MB; commits happen under the arena + lock — bounded, rare, deliberate. +* Linux-only elasticity; elsewhere the arena is plain fixed memory. +* Tested back to kernel 5.4, on 4 K, THP and hugetlb backings, x86_64. diff --git a/mem/common.c b/mem/common.c index 598b3516005..da33287f38c 100644 --- a/mem/common.c +++ b/mem/common.c @@ -25,7 +25,11 @@ #include "common.h" #include "../dprint.h" +#ifdef DBG_MALLOC enum osips_mm mem_allocator = MM_Q_MALLOC_DBG; +#else +enum osips_mm mem_allocator = MM_Q_MALLOC; +#endif /* returns -1 if @mm_name is unrecognized */ int set_global_mm(const char *mm_name) @@ -65,6 +69,19 @@ int parse_mm(const char *mm_name, enum osips_mm *mm) return 0; } + /* Both spellings select the same allocator. The binary names ITSELF + * with the generation suffix (mm_str(), opensips -V), so a log or a -V + * paste says which arena is running; but the plain name has to keep + * selecting it, because every /etc/default/opensips in the fleet passes + * "-a HG_MALLOC" and swapping a binary must not require editing the + * sizing file in lockstep - nor break the rollback in the other + * direction. See mem/hg_version.h. */ + if (!strcasecmp(mm_name, "HG_MALLOC") || + !strcasecmp(mm_name, HG_MALLOC_NAME)) { + *mm = MM_HG_MALLOC; + return 0; + } + #ifdef DBG_MALLOC if (!strcasecmp(mm_name, "F_MALLOC_DBG")) { *mm = MM_F_MALLOC_DBG; @@ -85,6 +102,12 @@ int parse_mm(const char *mm_name, enum osips_mm *mm) *mm = MM_F_PARALLEL_MALLOC_DBG; return 0; } + + if (!strcasecmp(mm_name, "HG_MALLOC_DBG") || + !strcasecmp(mm_name, HG_MALLOC_NAME "_DBG")) { + *mm = MM_HG_MALLOC_DBG; + return 0; + } #endif return -1; diff --git a/mem/common.h b/mem/common.h index ccba6d5e3b1..216a52dbdb7 100644 --- a/mem/common.h +++ b/mem/common.h @@ -37,14 +37,16 @@ extern gen_lock_t *hash_locks[TOTAL_F_PARALLEL_POOLS]; #include "meminfo.h" -#if !defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC) +#if !defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(HP_MALLOC) && \ + !defined(F_PARALLEL_MALLOC) && !defined(HG_MALLOC) #error "no memory allocator selected"b /* if exactly one allocator was selected, let's inline it! */ -#elif ((!defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(HP_MALLOC)) || \ - (!defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(F_PARALLEL_MALLOC)) || \ - (!defined(F_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC)) || \ - (!defined(Q_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC))) +#elif ((!defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC)) || \ + (!defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(HP_MALLOC) && !defined(HG_MALLOC)) || \ + (!defined(F_MALLOC) && !defined(Q_MALLOC) && !defined(F_PARALLEL_MALLOC) && !defined(HG_MALLOC)) || \ + (!defined(F_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC) && !defined(HG_MALLOC)) || \ + (!defined(Q_MALLOC) && !defined(HP_MALLOC) && !defined(F_PARALLEL_MALLOC) && !defined(HG_MALLOC))) #define INLINE_ALLOC #endif @@ -55,10 +57,12 @@ enum osips_mm { MM_Q_MALLOC, MM_HP_MALLOC, MM_F_PARALLEL_MALLOC, + MM_HG_MALLOC, MM_F_MALLOC_DBG, MM_Q_MALLOC_DBG, MM_HP_MALLOC_DBG, MM_F_PARALLEL_MALLOC_DBG, + MM_HG_MALLOC_DBG, }; #if defined F_MALLOC @@ -77,6 +81,15 @@ enum osips_mm { #include "f_parallel_malloc.h" #endif +#if defined HG_MALLOC +#include "hg_malloc.h" +#endif + +/* the allocator's own name for itself; see mem/hg_version.h. Included + * unconditionally so that mm_str() below spells MM_HG_MALLOC the same way in + * every build, including the ones that did not compile the arena in. */ +#include "hg_version.h" + extern int mem_warming_enabled; extern char *mem_warming_pattern_file; extern int mem_warming_percentage; @@ -94,10 +107,12 @@ int parse_mm(const char *mm_name, enum osips_mm *mm); (mm) == MM_Q_MALLOC ? "Q_MALLOC" : \ (mm) == MM_HP_MALLOC ? "HP_MALLOC" : \ (mm) == MM_F_PARALLEL_MALLOC ? "F_PARALLEL_MALLOC" : \ + (mm) == MM_HG_MALLOC ? HG_MALLOC_NAME : \ (mm) == MM_F_MALLOC_DBG ? "F_MALLOC_DBG" : \ (mm) == MM_Q_MALLOC_DBG ? "Q_MALLOC_DBG" : \ (mm) == MM_F_PARALLEL_MALLOC_DBG ? "F_PARALLEL_MALLOC_DBG" : \ - (mm) == MM_HP_MALLOC_DBG ? "HP_MALLOC_DBG" : "unknown") + (mm) == MM_HP_MALLOC_DBG ? "HP_MALLOC_DBG" : \ + (mm) == MM_HG_MALLOC_DBG ? HG_MALLOC_NAME "_DBG" : "unknown") #ifdef DBG_MALLOC typedef void *(*osips_block_malloc_f) (void *block, unsigned long size, diff --git a/mem/hg_arena.c b/mem/hg_arena.c new file mode 100644 index 00000000000..4609a3b014c --- /dev/null +++ b/mem/hg_arena.c @@ -0,0 +1,2045 @@ +/* + * hugepage-backed slab allocator - arena internals + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef HG_MALLOC + +#include +#include + +#include "hg_malloc.h" +#include "hg_arena.h" +#include "hg_buddy.h" +#include "hg_large.h" +#include "../dprint.h" +#include "../globals.h" + +/* ~x1.5 ladder, all multiples of 32 so cells stay HG_ROUNDTO-aligned; ported + * unchanged from cachedb_perf's pcache_arena.c cell_sizes[] - these are + * TOTAL slot sizes (header + payload), same convention as there */ +static const unsigned int cell_sizes[HG_NCLASSES] = { + 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, + 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, 49152, 65536 +}; + +#define HG_CHUNK_SMALL (256 * 1024) /* ceiling on chunk_max, see below */ +#define HG_CHUNK_MIN (8 * 1024) /* floor, however tiny the arena */ +/* + * Target cells per block. A block is the unit of reclaim, so this is a + * fragmentation knob, not an amortisation one: fewer cells per block means a + * block drains sooner, more means fewer carves. 32 is the design's figure and + * puts class 2048 on a 64 KB block; do not raise it without re-measuring what + * fraction of blocks actually reach empty. + */ +#define HG_CELLS_PER_BLOCK 32 +/* + * Drained blocks kept per class rather than returned. + * + * ZERO, deliberately, after measuring: with concentration in place a refill + * drives partial blocks to no free cells at all, so gpool_pop() falls back to + * the one drained block and un-drains it - and with a keep of 1 the queue + * could never reach the 2 entries the collector waited for. Reclaim went from + * 1 block returned to 0. The hysteresis was fighting the thing it sits in + * front of. + * + * Keeping none costs less than it appears: a returned block goes onto the + * buddy's free list at its own order, so carve_chunk() can take the very same + * block straight back with no split and no merge. The buddy IS the cache, and + * holding a block back from it only hides the memory. + */ +#define HG_GC_KEEP 0 +#define HG_REFILL_BATCH 32 /* cells pulled from the global pool */ + +/* + * Private free-cache budget. + * + * This used to be a flat count - 256 cells per class, whatever the class - + * which bounds nothing, because the same number means 16 KB in class 64 and + * 16 MB in class 65536. One thread could therefore hoard 16 MB of a single + * class, and nothing but an arena smaller than the ceiling stopped it. + * + * The bound that matters is BYTES, so express it that way and convert to a + * per-class cell count at init. Two numbers define the policy: + * + * HG_PRIVATE_PCT what share of the arena ALL private caches together + * may hold - the whole fleet of threads, every class; + * HG_PRIVATE_CONSUMERS how many threads to divide that share between. + * + * So one thread gets PCT/CONSUMERS of the arena (25/16 ~ 1.6%), and the + * aggregate stays at PCT by construction however many classes exist. + * Measured on the real ladder: shm 256 MB gives a thread at most 3.0 MB + * across all 21 classes, and pkg 24 MB gives it 0.29 MB. The class that + * motivated this - 65536 - drops from 256 cells (16 MB!) to 3 on shm and to + * none at all on pkg. + * + * The binding case is PKG, not SHM, and it is new in 4.1: pkg is MAP_PRIVATE + * per process, and TCP main is ONE process running an IO thread pool (one + * thread per CPU by default) against a single 8 MB pkg arena. Sixteen threads + * each hoarding "a few cells of every class" is how an 8 MB arena disappears + * into caches that no other thread can reach. Budgeting for the pool size + * keeps the total bounded by construction: worst case is + * HG_PRIVATE_PCT percent of the arena, however many classes exist. + * + * HG_PRIVATE_CAP keeps the old ceiling for the small classes, where the byte + * budget alone would now permit MORE hoarding than before on a large arena - + * this change is meant to lower the bound, never raise it. + */ +#define HG_PRIVATE_PCT 25 /* of the arena, per thread, all classes */ +#define HG_PRIVATE_CONSUMERS 16 /* budget for a per-CPU IO pool */ +#define HG_PRIVATE_CAP 256 /* never above the historical flat cap */ + +/* + * Per-THREAD private free-stack state, one slot per hg_block instance live + * in this thread (see the hg_arena.h comment on why this can't be the + * single static global cachedb_perf uses: HG_MALLOC can back shm, shm_dbg + * AND pkg simultaneously). A fixed-size static array, not heap-allocated - + * this state is exactly the kind of bootstrap-before-any-allocator-exists + * bookkeeping that must NOT go through pkg_malloc()/shm_malloc(), since + * HG_MALLOC may itself be backing one or both of those. + * + * __thread, NOT merely static, and that is a correctness requirement rather + * than a tuning choice. The whole fast path is lock-free precisely because + * this state is private to its owner; a plain static makes it private to the + * PROCESS, which was true of OpenSIPS's classic one-thread-per-process model + * but is NOT true since 4.1 - TCP main now runs a pthread IO pool + * (tcp_pool_init(), net_tcp.c, one thread per CPU by default, started + * unconditionally), and those threads run the read callbacks that reach + * tcp_dispatch_msg() -> shm_malloc(). Several threads therefore hit these + * lists at once. + * + * Observed on real traffic before this was made per-thread: + * - lost "nfree--" updates, so the counter UNDERFLOWED and wrapped + * (logged: "nfree claimed 4294966733", i.e. 2^32-563), which trivially + * exceeds any sane private-cache bound and drove the donation loop off + * the end of a + * chain that had far fewer cells than claimed; + * - two threads popping the SAME cell, handing one block to two callers - + * a tcp_ipc_payload struct came back with SIP text where its conn + * pointer belonged, and tcpconn_put() died on it. + * Both faults were the same race wearing different masks. si_addr on the + * first is 0x20 == HG_CELL_HDR, i.e. cell_next(NULL). + * + * Cost: sizeof(struct hg_palloc) is ~512B, so ~2KB of TLS per thread for all + * four instances - trivial next to what it buys, and the fast path stays + * lock-free rather than gaining a mutex. + * + * Note what stays process-wide on purpose: hb->gpool (the shared pool, taken + * under hb->lock) is unchanged, so cells still circulate between threads and + * processes normally. A thread that exits leaves its cached cells parked in + * its own dead TLS rather than returning them - harmless here, because the + * IO pool threads live for the lifetime of the process. + */ +#define HG_MAX_INSTANCES 4 +static __thread struct hg_palloc palloc_slots[HG_MAX_INSTANCES]; + +/* + * One-entry lookup cache for the slot this thread used last. + * + * Every alloc and every free begins by resolving hb -> palloc, so this is + * the single hottest lookup in the allocator, and it was a linear walk of + * HG_MAX_INSTANCES slots comparing owner pointers - on EVERY call, since + * before this the only way in was the slow path below. In practice a worker + * bounces between two instances at most (shm and pkg), and overwhelmingly + * hits the same one many times in a row, so the walk almost always finds its + * answer after re-testing slots it just rejected. + * + * Splitting it lets the fast path inline into hg_cell_alloc()/hg_cell_free() + * as a TLS load and one compare, while the cold path stays out of line. That + * matters beyond the instruction count: the slow path carries a stack- + * protector prologue (GCC adds one once __thread and the memset() are in + * play), and an inlined fast path skips the call and the canary entirely + * rather than paying them per allocation. + * + * Staleness is safe by construction: the cache is only trusted when + * pl->owner still equals the requested block. hg_arena_child_init() and + * hg_arena_destroy() zero a slot's owner, so a cached pointer to a recycled + * slot simply fails that test and falls through to the walk - but both also + * clear hg_last outright, so the invalidation is explicit rather than + * relying on that. + */ +static __thread struct hg_palloc *hg_last; + +static struct hg_palloc *hg_palloc_lookup(struct hg_block *hb) +{ + int i, free_slot = -1; + + for (i = 0; i < HG_MAX_INSTANCES; i++) { + if (palloc_slots[i].owner == hb) { + hg_last = &palloc_slots[i]; + return &palloc_slots[i]; + } + if (free_slot < 0 && !palloc_slots[i].owner) + free_slot = i; + } + + if (free_slot < 0) { + LM_CRIT("more than %d live HG_MALLOC instances in one process " + "- raise HG_MAX_INSTANCES in hg_arena.c\n", HG_MAX_INSTANCES); + return NULL; + } + + memset(&palloc_slots[free_slot], 0, sizeof(struct hg_palloc)); + palloc_slots[free_slot].owner = hb; + hg_last = &palloc_slots[free_slot]; + return &palloc_slots[free_slot]; +} + +static inline struct hg_palloc *hg_get_palloc(struct hg_block *hb) +{ + struct hg_palloc *pl = hg_last; + + if (pl && pl->owner == hb) + return pl; + return hg_palloc_lookup(hb); +} + +/* + * Free-list link storage: a SINGLE convention, used everywhere (private + * free-stack and the shared global pool alike) - both helpers take + * @cell_start (the address of the hidden header / class byte), and write + * the link at cell_start + HG_CELL_HDR, i.e. inside the PAYLOAD area, never + * inside the header itself. This is safe because nobody reads the payload + * of a free cell, and it keeps the class byte at offset 0 untouched no + * matter which free list (private or shared) currently owns the cell. + */ +static inline void *cell_next(void *cell_start) +{ + return *(void **)((char *)cell_start + HG_CELL_HDR); +} + +static inline void cell_set_next(void *cell_start, void *next) +{ + *(void **)((char *)cell_start + HG_CELL_HDR) = next; +} + +/* + * Does @cell_start belong to THIS arena? + * + * Everything hg hands out - small cells in chunks and large frags alike - + * is carved from the single hb->hbase reservation, whose bounds are fixed + * at init. Deliberately NOT hb->lo/hb->hi: those widen as chunks are + * carved, so a lock-free reader can see a stale, too-narrow range and + * reject a perfectly good cell. + * + * Why this matters: a process routinely hosts THREE live hg instances (shm, + * pkg, and cachedb_perf's own arena), all using an identical cell layout. + * A pointer from one of the others carries a class byte that looks entirely + * valid here (0..HG_NCLASSES-1), so without this check it would sail past + * every test below and land on THIS arena's private free list. Its real + * owner then reuses the cell and overwrites the payload - which is exactly + * where cell_set_next() keeps the free-list link - truncating our chain + * while nfree keeps counting the cells that were on it. That drift is what + * later drove the donation loop off the end of the chain and into + * cell_next(NULL). Refusing the free leaks one cell; accepting it corrupts + * the pool. + */ +/* global pool ops - hb->lock must be held. Both take/return cell_start. */ +/* --- shared free cells, held per block and graded by fullness ---------- */ + +/* + * Which fullness list a block belongs on, from its own free-cell count. + * Grade 0 is the FULLEST partial (fewest free cells), which is what a refill + * wants: draining the fullest block leaves the emptier ones alone to reach + * zero, instead of topping every block up a little. + */ +static inline unsigned int grade_of(const struct hg_chunk *ch) +{ + unsigned int g; + + if (ch->in_gpool == 0) + return HG_GRADE_NONE; + if (ch->in_gpool >= ch->cells) + return HG_GRADE_DRAINED; + g = (ch->in_gpool * HG_GRADES) / ch->cells; + if (g >= HG_GRADES) + g = HG_GRADES - 1; + return g; +} + +static inline void list_unlink(struct hg_block *hb, int c, struct hg_chunk *ch) +{ + struct hg_chunk **head; + + if (ch->grade == HG_GRADE_NONE) + return; + head = (ch->grade == HG_GRADE_DRAINED) ? &hb->drained[c] + : &hb->bucket[c][ch->grade]; + if (ch->fprev) + ch->fprev->fnext = ch->fnext; + else + *head = ch->fnext; + if (ch->fnext) + ch->fnext->fprev = ch->fprev; + ch->fnext = ch->fprev = NULL; + if (ch->grade == HG_GRADE_DRAINED) { + hb->ndrained[c]--; + ch->flags &= ~HG_CHUNK_DRAINED; + } + ch->grade = HG_GRADE_NONE; +} + +static inline void list_link(struct hg_block *hb, int c, struct hg_chunk *ch, + unsigned int g) +{ + struct hg_chunk **head; + + if (g == HG_GRADE_NONE) { + ch->grade = HG_GRADE_NONE; + return; + } + head = (g == HG_GRADE_DRAINED) ? &hb->drained[c] : &hb->bucket[c][g]; + ch->fprev = NULL; + ch->fnext = *head; + if (ch->fnext) + ch->fnext->fprev = ch; + *head = ch; + ch->grade = g; + if (g == HG_GRADE_DRAINED) { + hb->ndrained[c]++; + ch->flags |= HG_CHUNK_DRAINED; + } +} + +/* move @ch to the list its current fullness calls for; a no-op when the grade + * has not changed, which is the common case - a block crosses a grade + * boundary far less often than it gains or loses a cell */ +static inline void block_regrade(struct hg_block *hb, int c, + struct hg_chunk *ch) +{ + unsigned int g = grade_of(ch); + + if (g == ch->grade) + return; + list_unlink(hb, c, ch); + list_link(hb, c, ch, g); +} + +/* + * Resolve a cell to its block, and refuse anything that is not a class chunk + * of THIS class. The large tier allocates buddy blocks too, and those start + * with a struct hg_large_chunk - writing in_gpool there would land inside its + * first_frag pointer and corrupt the large heap. Only class cells reach here + * today, so this should never trip; it is present because the cost of being + * wrong is silent corruption of a different allocator tier. + */ +static inline struct hg_chunk *cell_block(struct hg_block *hb, int c, void *p) +{ + struct hg_chunk *ch = hg_buddy_block_of(hb, p); + + if (!ch) + return NULL; + if (ch->cls != (unsigned int)c) { + hg_corrupt(hb, HG_C_CLASS_MISMATCH); + LM_CRIT("%s: cell %p resolves to block %p of class %u, expected " + "class %d - not touching it\n", + hb->name, p, (void *)ch, ch->cls, c); + return NULL; + } + return ch; +} + +/* + * Hand fully-drained blocks of class @c back to the buddy, keeping the last + * HG_GC_KEEP as hysteresis. + * + * With per-block free lists this is O(1) per block: every cell of a drained + * block is on that block's OWN list, so there is nothing to unlink from a + * shared structure - the list is simply discarded with the block. The + * previous shape had to walk the whole per-class free list pulling the + * block's cells out of it. + * + * hb->lock must be held. + */ +static void gc_class(struct hg_block *hb, int c) +{ + struct hg_chunk *ch; + unsigned int freed = 0; + + while (hb->ndrained[c] > HG_GC_KEEP) { + unsigned int ord; + unsigned long sz; + + /* take from the tail-most entry we can reach cheaply: the head is + * the most recently drained, which is the one worth keeping */ + ch = hb->drained[c]; + while (ch->fnext) + ch = ch->fnext; + + list_unlink(hb, c, ch); + hb->gpool_n[c] -= ch->in_gpool; + ch->in_gpool = 0; + ch->free_head = NULL; + + /* out of the registry, or hg_slab_recycled() keeps counting a + * capacity that no longer exists and the DBG walker reads a block + * the buddy has since handed to another class */ + if (ch->prev) + ch->prev->next = ch->next; + else + hb->chunks = ch->next; + if (ch->next) + ch->next->prev = ch->prev; + hb->nchunks--; + + ord = ch->order; + sz = HG_LEAF_SIZE << ord; + hb->real_used -= sz; + + /* every read of ch must precede this: the buddy immediately reuses + * the block's first bytes for its own free-list linkage */ + hg_buddy_free(hb, ch, ord); + freed++; + } + + if (freed) { + hb->gc_blocks_returned += freed; + hb->gc_passes++; + LM_DBG("%s gc class %d: returned %u blocks, %u drained kept\n", + hb->name, c, freed, hb->ndrained[c]); + } +} + +/* a cell becomes shared: onto its OWN block's free list */ +static inline void gpool_push(struct hg_block *hb, int c, void *cell_start) +{ + struct hg_chunk *ch = cell_block(hb, c, cell_start); + + if (!ch) + return; /* refused above, with a CRIT */ + if (ch->in_gpool >= ch->cells) { + hg_corrupt(hb, HG_C_DOUBLE_FREE); + LM_CRIT("%s: block %p already has all %u cells free, refusing to " + "add another - this is a double free\n", + hb->name, (void *)ch, ch->cells); + return; + } + cell_set_next(cell_start, ch->free_head); + ch->free_head = cell_start; + ch->in_gpool++; + hb->gpool_n[c]++; + block_regrade(hb, c, ch); + + /* + * Deferred during a cache flush, and that is a correctness requirement, + * not a tuning one. The flush walks a chain of cached cells; pushing one + * can complete its block and hand it to the buddy, which immediately + * writes free-list linkage over its first bytes. If the NEXT cell on the + * chain belongs to that same block - and cells of one block are exactly + * what a flush tends to hold - the walk would then dereference recycled + * memory. Collect first, collect the reclaim afterwards. + */ + if (!hb->gc_deferred && hb->ndrained[c] > HG_GC_KEEP) + gc_class(hb, c); +} + +/* + * Take a shared cell, from the FULLEST partial block - the concentration the + * whole reclaim depends on. Scanning grades upward from 0 finds it in at most + * HG_GRADES pointer tests. + */ +static inline void *gpool_pop(struct hg_block *hb, int c) +{ + struct hg_chunk *ch = NULL; + void *cell_start; + unsigned int g; + + for (g = 0; g < HG_GRADES; g++) + if (hb->bucket[c][g]) { + ch = hb->bucket[c][g]; + break; + } + /* nothing partial - reuse a drained block rather than carve a fresh + * one; it is already ours and already the right class */ + if (!ch) + ch = hb->drained[c]; + if (!ch) + return NULL; + + cell_start = ch->free_head; + if (!cell_start) { + hg_corrupt(hb, HG_C_NFREE_UNDERFLOW); + LM_CRIT("%s: block %p claims %u free cells but its list is empty - " + "dropping it from the pool\n", + hb->name, (void *)ch, ch->in_gpool); + hb->gpool_n[c] -= ch->in_gpool; + ch->in_gpool = 0; + block_regrade(hb, c, ch); + return NULL; + } + ch->free_head = cell_next(cell_start); + ch->in_gpool--; + hb->gpool_n[c]--; + block_regrade(hb, c, ch); + return cell_start; +} + +/* + * Give up every cell this thread has cached, for every class. + * + * This is the only way those cells can ever be seen again: they live in + * __thread TLS, so no other process or thread can reach them - which is why + * the design forbids a central sweeper and requires the flush to run ON the + * owning thread. Measured, this is what stands between the reclaim and the + * arena: blocks stalled at 37 of 42 cells with the remainder sitting here. + * + * hb->lock must be held. Returns the number of cells handed over. + */ +static unsigned int cache_flush_locked(struct hg_block *hb, + struct hg_palloc *pl) +{ + unsigned int c, n = 0; + + if (!pl) + return 0; + + hb->gc_deferred = 1; + for (c = 0; c < HG_NCLASSES; c++) { + void *cur = pl->cls[c].free_head; + + while (cur) { + void *nxt = cell_next(cur); + + gpool_push(hb, c, cur); + cur = nxt; + n++; + } + pl->cls[c].free_head = NULL; + pl->cls[c].nfree = 0; + } + hb->gc_deferred = 0; + + /* now it is safe to let blocks go - nothing is walking their cells */ + for (c = 0; c < HG_NCLASSES; c++) + if (hb->ndrained[c] > HG_GC_KEEP) + gc_class(hb, c); + + return n; +} + +/* + * Flush THIS thread's caches, in every arena it holds cache state for. + * + * The entry point the idle sweep dispatches to. It takes no block argument on + * purpose: the caller (a timer, or an IPC job running in some worker) has no + * business knowing which arenas exist, and palloc_slots[] already records + * exactly the set this thread caches in - shm, pkg, and the debug arenas if + * they are live. + * + * Must run ON the owning thread. That is not a preference: the caches are + * __thread, so no other process or thread can even address them, which is why + * the design rules out a central sweeper and why the dispatcher has to make + * each worker do its own (and call this inline for itself rather than sending + * itself an IPC job to order against - see signal_pkg_status()). + */ +/* + * Sweep generation. Bumped once per sweep by the dispatcher; a thread that + * cannot be reached by IPC compares its own last-seen value against it at a + * job boundary and flushes when they differ. + * + * This exists for exactly one caller: TCP main's IO pool. Those threads wait + * on a condition variable rather than the reactor, so ipc_send_rpc() has no + * way to reach them - and they are the worst case for a stranded cache, + * because pkg is MAP_PRIVATE per process and that pool runs one thread per + * CPU against a single 8 MB arena. + * + * A plain counter, deliberately not a lock or a handshake: a missed + * generation only delays a flush to the next sweep, which is the same + * fire-and-forget contract the IPC path already has. + */ +volatile unsigned long hg_sweep_gen; + +/* the two checks that fire where no arena pointer is in scope */ +unsigned long hg_corrupt_noarena[HG_CORRUPT_KINDS]; + +/* + * Reserve floor: when free grid space falls below hb->reserve_floor (1/16 of + * the grid, set in hg_buddy_init), publish a sweep to every thread and say so + * once. Crossing it bumps hg_sweep_gen, which reaches even the TCP IO pool - + * IPC cannot. The below_floor latch is the hysteresis: without it a workload + * sitting on the boundary would log and re-sweep on every single allocation, + * which is both useless and expensive exactly when the arena is under + * pressure. Recovery needs a 2x margin so it cannot flap. + * + * THIS MUST BE CALLED FROM THE BUDDY LAYER, not from its callers. It used to + * live inline in carve_chunk(), which meant it only ever saw the SLAB carve + * path - and the large tier takes grid space directly via hg_buddy_alloc() + * and hg_buddy_alloc_run(), as does hg_region_alloc(). A burst of large + * allocations could therefore drive free leaves from 3303 to 51 against a + * floor of 256 - five times past it - with floor_crossings still reading 0 and + * no warning logged. Measured exactly that on 2026-08-11 before this moved. + * + * Calling it from hg_buddy_alloc/_free instead puts it on the one choke point + * every consumer must pass, so a consumer added later cannot silently skip it, + * which is precisely how it was missed the first time. Called on the free path + * too: recovery is a rise in free leaves, and nothing on the alloc path can + * observe that. + * + * hb->lock is held by every caller. + */ +/* + * Widen the [lo, hi] extent watermarks to cover a region just handed out. + * + * These bound "a pointer this arena could plausibly have returned", and the + * DBG free guard in hg_malloc_dyn.h ABORTS on anything outside them. They were + * widened in exactly two places - carve_chunk() and hg_region_alloc() - both + * written when carving was the only way to get memory. The large tier takes + * its backing straight from hg_buddy_alloc()/_alloc_run() and never touched + * them, so a large fragment served from grid space above the current hi was a + * perfectly valid pointer that the guard killed the process over. + * + * Observed 2026-08-11: tm freeing a cloned 65 KB request (a large-tier + * allocation) at 0x...66100078 against hi 0x...660e0000 - 131,192 bytes past + * it, inside the arena by hbase+hsize, and hg_owns() agreed it was ours. + * + * Lives at the buddy layer for the same reason hg_reserve_floor_check() does: + * it is the one point every consumer of grid space must pass, so a consumer + * added later cannot silently skip it. That is the third time this exact shape + * has bitten - real_used, the reserve floor, and now these. + * + * hb->lock is held by every caller. + */ +void hg_extent_note(struct hg_block *hb, void *base, unsigned long size) +{ + unsigned long b = (unsigned long)base; + + if (b < hb->lo) + hb->lo = b; + if (b + size > hb->hi) + hb->hi = b + size; +} + +void hg_reserve_floor_check(struct hg_block *hb) +{ + if (!hb->buddy_ready || !hb->reserve_floor) + return; + + if (hb->buddy_free_leaves < hb->reserve_floor) { + if (!hb->below_floor) { + hb->below_floor = 1; + hb->floor_crossings++; + hg_sweep_gen++; + LM_WARN("%s: free space fell below the reserve floor " + "(%lu of %lu leaves free, floor %lu) - sweeping every " + "thread's cache; raise -m/-M if this repeats\n", + hb->name, hb->buddy_free_leaves, + hb->npages * hg_leaves_per_page(hb), hb->reserve_floor); + } + } else if (hb->below_floor && + hb->buddy_free_leaves > hb->reserve_floor * 2) { + hb->below_floor = 0; + LM_NOTICE("%s: free space recovered above the reserve floor " + "(%lu leaves free)\n", hb->name, hb->buddy_free_leaves); + /* recovery past the same 2x-floor threshold also ends a + * grow-blocked episode: the arena could not grow, but the + * demand that needed it to has gone away. One hysteresis + * mark for both states, deliberately - two thresholds + * drifting apart would let "blocked" outlive the pressure + * that defined it. */ + hg_grow_unblock(hb, "demand fell back below the floor"); + } +} + +static __thread unsigned long hg_sweep_seen; + +void hg_cache_flush_if_due(void) +{ + unsigned long g = hg_sweep_gen; + + if (g == hg_sweep_seen) + return; + hg_sweep_seen = g; + hg_cache_flush_self(); +} + +void hg_cache_flush_self(void) +{ + int i; + + for (i = 0; i < HG_MAX_INSTANCES; i++) { + struct hg_block *hb = palloc_slots[i].owner; + unsigned int n; + + if (!hb) + continue; + lock_get(&hb->lock); + n = cache_flush_locked(hb, &palloc_slots[i]); + hb->cache_flushes++; + hb->cells_flushed += n; + /* v3: a PRIVATE arena's shrink gate ticks here - only its + * owning process can release its memory, and this runs in + * every process once per sweep. The shared arena is ticked + * by the sweep timer alone, or 30 workers would each tick + * the one shared window counter. */ + if (!hb->shared) { + hg_grow_tick(hb); + hg_shrink_tick(hb); + } + lock_release(&hb->lock); + if (n) + LM_DBG("%s: idle sweep returned %u cached cells\n", hb->name, n); + } +} + +/* + * Every byte of arena memory funnels through here: bump the atomic offset + * within the block's own huge-page reservation. No fallback to another + * allocator on exhaustion (unlike cachedb_perf's shm_malloc() fallback) - + * per the HG_MALLOC design decision, exhaustion is a hard, loud failure; + * the operator increases -m/-M instead of silently landing on 4K pages + * mid-run through a different allocator's pool. + */ +void *hg_chunk_backing(struct hg_block *hb, unsigned long size) +{ + unsigned long asz = (size + 63) & ~63UL; /* keep 64-aligned */ + unsigned long off; + + /* + * Init only, once the buddy owns the arena. + * + * hg_buddy_init() reserves every leaf below hoff and publishes the rest + * as free. A bump AFTER that returns memory the buddy already considers + * free, so the same bytes get handed to two owners - silently, and with + * a delay before the corruption shows. There is exactly one legitimate + * caller left (the buddy carving its own metadata, before it is ready), + * so anything else is a bug and says so rather than corrupting. + */ + if (hb->buddy_ready) { + hg_corrupt(hb, HG_C_INTERNAL); + LM_CRIT("%s: bump carve of %lu bytes after the buddy owns the " + "arena - refusing, this would double-allocate\n", + hb->name, size); + return NULL; + } + + off = __atomic_fetch_add(&hb->hoff, asz, __ATOMIC_RELAXED); + + if (off + asz <= hb->hsize) + return hb->hbase + off; + + /* exhausted: undo would race other bumpers, so just leave hoff past + * the end (further allocs also fail) - correctness holds, we only + * lose the tail slack */ + return NULL; +} + +/* + * As hg_chunk_backing(), but the returned address is @align-aligned. + * + * The buddy layer needs whole huge pages at their natural alignment, and + * over-allocating by align-1 to trim afterwards would throw away up to a + * whole 2 MB page per page claimed. So this bumps by the EXACT padded amount + * under a compare-exchange instead: the pad is computed from the candidate + * offset, and if another bumper wins the race the pad is recomputed against + * the new offset rather than reused. + * + * Unlike hg_chunk_backing() a failed reservation does not consume the tail - + * the CAS simply never commits - so an oversized request cannot poison the + * arena for the smaller ones behind it. + * + * @align must be a power of two. Alignment is applied to the ADDRESS, not to + * the offset, because hbase itself is not guaranteed aligned on the non-Linux + * reserve path. + */ +void *hg_backing_aligned(struct hg_block *hb, unsigned long size, + unsigned long align) +{ + unsigned long asz = (size + 63) & ~63UL; + unsigned long cur, aligned_off, newoff; + + if (align < 64) + align = 64; + if (align & (align - 1)) { + LM_ERR("%s: alignment %lu is not a power of two\n", hb->name, align); + return NULL; + } + + cur = __atomic_load_n(&hb->hoff, __ATOMIC_RELAXED); + do { + unsigned long addr = (unsigned long)hb->hbase + cur; + + aligned_off = cur + ((~addr + 1) & (align - 1)); + newoff = aligned_off + asz; + if (newoff > hb->hsize) + return NULL; + } while (!__atomic_compare_exchange_n(&hb->hoff, &cur, newoff, 1, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + return hb->hbase + aligned_off; +} + +/* + * Chunk granularity has to scale with the arena, not be a fixed 256K. + * + * A chunk is claimed whole the first time its size class is touched, so + * with a fixed size the 21 classes cost 21 * 256K = 5.25M of granularity + * before a single useful byte is served - which an 8M pkg arena cannot + * afford, and it then fails to parse SIP messages at all. (F_MALLOC has no + * equivalent floor because it splits fragments to fit.) Cap a chunk at a + * small fraction of the arena so a small arena gets proportionally small + * chunks, while a large one keeps the full 256K and its amortisation. + */ +static inline unsigned int chunk_size_for(struct hg_block *hb, int c) +{ + unsigned int want, least = sizeof(struct hg_chunk) + cell_sizes[c] * 2; + + /* + * Aim for HG_CELLS_PER_BLOCK cells, floored at one buddy leaf. + * + * This deliberately REPLACES the old "256 KB for every class up to + * 8 KB cells". A block is the unit of reclaim now, and a block only + * comes back when every one of its cells is free, so a block holding + * 2730 cells of class 96 is a block that will essentially never drain - + * one survivor pins 256 KB. That is exactly the "too coarse" failure + * the design rejects whole-chunk reclaim for. + * + * The sizes fall where the design says: class 96 lands on the 8 KB + * floor (85 cells), class 2048 on 64 KB (32 cells) rather than an 8 KB + * block holding only four. + */ + want = cell_sizes[c] * HG_CELLS_PER_BLOCK; + if (want < HG_LEAF_SIZE) + want = HG_LEAF_SIZE; + + if (want > hb->chunk_max) + want = hb->chunk_max; + /* ...but always enough for the header plus a couple of cells, or the + * class could never be served at all */ + if (want < least) + want = least; + return want; +} + +/* carve a new chunk for class @c - hb->lock must be held. The class byte of + * every cell's hidden header is stamped HERE, before the chunk is reachable + * by anyone - immutable from birth. */ +static int carve_chunk(struct hg_block *hb, int c, struct hg_palloc *pl) +{ + struct hg_chunk *ch; + unsigned int size = chunk_size_for(hb, c), i; + char *cells; + int ord; + + /* + * Chunks are buddy blocks now, so the size rounds UP to an order. That + * is not waste: a chunk is a bag of cells, so a bigger block simply + * holds more of them, and in exchange the block is naturally aligned + * and - once the GC lands - returnable. The bump allocator it replaces + * could never give any of that back. + */ + ord = hg_buddy_order_for(hb, size); + if (ord < 0) { + LM_ERR("%s: class %d wants a %u byte chunk, larger than the %lu byte " + "page the buddy tops out at\n", hb->name, c, size, + (unsigned long)hb->hps); + return -1; + } + size = (unsigned int)(HG_LEAF_SIZE << ord); + ch = hg_buddy_alloc(hb, (unsigned int)ord); + /* v3: exhaustion is a growth trigger before it is an error. One retry + * only - if the arena grew, the freshly published pages satisfy this + * order by construction (they are whole), so a second miss can only + * mean the grow itself was refused and would be refused again. */ + if (!ch && hg_buddy_grow(hb, size) == 0) + ch = hg_buddy_alloc(hb, (unsigned int)ord); + + /* + * Reserve floor. Checked HERE, at the one point where the arena's free + * space actually shrinks, rather than on a timer - the design's "trigger + * the sweep on reserve pressure, not only on time". + * + * Crossing it publishes a sweep for every thread (the generation counter + * reaches even the TCP IO pool, which IPC cannot) and says so once. The + * below_floor latch is the hysteresis: without it a workload sitting on + * the boundary would log and re-sweep on every single carve, which is + * both useless and expensive exactly when the arena is under pressure. + */ + /* the floor is evaluated inside the buddy layer now - see + * hg_reserve_floor_check() for why it cannot live here */ + if (!ch) { + LM_ERR("%s: no more HG_MALLOC arena memory for a %u byte chunk " + "(class %d, order %d) - increase the arena size\n", + hb->name, size, c, ord); + return -1; + } + + /* the block comes from the buddy carrying whatever the free-list + * linkage left in its first bytes, so every field is set here, not + * assumed zero */ + ch->in_gpool = 0; + ch->flags = 0; + ch->order = (unsigned int)ord; + ch->free_head = NULL; + ch->grade = HG_GRADE_NONE; + ch->fnext = NULL; + ch->fprev = NULL; + ch->cls = c; + ch->cell_size = cell_sizes[c]; + ch->cells = (size - sizeof(struct hg_chunk)) / cell_sizes[c]; + + cells = (char *)ch + sizeof(struct hg_chunk); + for (i = 0; i < ch->cells; i++) + cells[(unsigned long)i * cell_sizes[c]] = (unsigned char)c; + + ch->next = hb->chunks; + ch->prev = NULL; + if (ch->next) + ch->next->prev = ch; + hb->chunks = ch; + hb->nchunks++; + hb->blocks_carved++; + hb->real_used += size; + if (hb->real_used > hb->max_real_used) + hb->max_real_used = hb->real_used; + + if ((unsigned long)ch < hb->lo) + hb->lo = (unsigned long)ch; + if ((unsigned long)ch + size > hb->hi) + hb->hi = (unsigned long)ch + size; + + /* the whole chunk belongs to the carving process */ + pl->cls[c].bump = cells; + pl->cls[c].left = ch->cells; + + LM_DBG("%s class %d: new %u byte chunk, %u cells of %u\n", + hb->name, c, size, ch->cells, cell_sizes[c]); + return 0; +} + +/* + * Turn the byte budget into a per-class cell count, once, at arena init. + * + * A class whose single cell already exceeds one thread's per-class share gets + * priv_max 0: it is never cached privately and every free goes straight to the + * shared pool. That is the correct answer rather than a degenerate one - those + * allocations are rare, so the lock they now take is rare too, and the memory + * they would have pinned is large. + * + * priv_donate is half the cap, so the donation loop can always be satisfied by + * a chain whose length matches nfree - the invariant the old fixed HG_DONATE + * relied on, now maintained per class instead of by two constants that had to + * be kept in the right order by hand. + */ +static void private_caps_init(struct hg_block *hb) +{ + unsigned long per_thread = (unsigned long)hb->size / 100 * HG_PRIVATE_PCT + / HG_PRIVATE_CONSUMERS; + unsigned long per_class = per_thread / HG_NCLASSES; + unsigned int c; + + for (c = 0; c < HG_NCLASSES; c++) { + unsigned long n = per_class / cell_sizes[c]; + + if (n > HG_PRIVATE_CAP) + n = HG_PRIVATE_CAP; + hb->priv_max[c] = (unsigned int)n; + hb->priv_donate[c] = (unsigned int)(n / 2); + if (hb->priv_donate[c] == 0) + hb->priv_donate[c] = 1; + } + + LM_DBG("%s: private cache budget %lu B/thread (%lu B/class): " + "class %u caches %u cells, class %u caches %u\n", + hb->name, per_thread, per_class, + cell_sizes[0], hb->priv_max[0], + cell_sizes[HG_NCLASSES - 1], hb->priv_max[HG_NCLASSES - 1]); +} + +/* + * Lay out the page grid the v2 buddy addresses through, and PROVE the + * arithmetic on real addresses before anything is built on top of it. + * + * Everything above the cell level resolves an address to its block with + * two shifts and a mask (README.hg_arena_v2, "Address to block"). That is + * only sound if the origin is huge-page aligned and the derived page and + * leaf indices round-trip. Getting it wrong would not fail loudly at the + * point of the mistake - it would hand out a block descriptor belonging to + * a different block, which surfaces much later as corruption, so it is + * checked here rather than assumed. + * + * Returns 0 when the grid is usable, -1 when the arithmetic does not hold. + */ +static int pages_init(struct hg_block *hb) +{ + unsigned long hps = hb->hps, off; + unsigned int shift = 0; + const char *end; + int i; + + if (!hps || (hps & (hps - 1))) { + LM_ERR("%s: huge page size %lu is not a power of two - cannot " + "build the page grid\n", hb->name, hps); + return -1; + } + while ((1UL << shift) < hps) + shift++; + hb->hps_shift = shift; + + /* + * Anchor the shift to the probed size. Everything below derives page + * indices FROM hps_shift and checks them against each other, so a shift + * that disagrees with hps is self-consistent and would sail through the + * probe loop - the corrupted-grid harness caught exactly that. This is + * the one comparison that ties the grid to physical reality. + */ + if ((1UL << shift) != hps) { + LM_ERR("%s: page shift %u describes %lu bytes, but the probed huge " + "page is %lu\n", hb->name, shift, 1UL << shift, hps); + return -1; + } + + if (shift <= HG_LEAF_SHIFT) { + LM_ERR("%s: huge page size %lu is not larger than the %lu byte " + "buddy leaf\n", hb->name, hps, HG_LEAF_SIZE); + return -1; + } + + /* page 0 starts at the first aligned address at or after hbase. On + * Linux every reserve path already aligns, so this is hbase and the + * subtraction below is zero; the non-Linux fallback is a plain mmap() + * and loses the unaligned head. */ + hb->pbase = (char *)(((unsigned long)hb->hbase + hps - 1) & ~(hps - 1)); + end = hb->hbase + hb->hsize; + hb->npages = (unsigned long)(end - hb->pbase) >> shift; + /* the grid's full extent runs to the CAP - descriptors for every page + * that could ever exist are laid out at init (hg_buddy_init), so a + * grow publishes pages instead of relocating metadata */ + hb->npages_cap = (unsigned long)(hb->hbase + hb->hcap - hb->pbase) + >> shift; + if (hb->npages_cap < hb->npages) /* hcap==hsize, or a tiny arena */ + hb->npages_cap = hb->npages; + + if (hb->pbase != hb->hbase) + LM_INFO("%s: reservation base %p is not %lu-aligned, losing %lu " + "bytes of head to align the page grid\n", hb->name, + hb->hbase, hps, (unsigned long)(hb->pbase - hb->hbase)); + + if (hb->npages == 0) { + /* An arena smaller than one huge page is legitimate (a tiny -M on + * a 512M-page arm64 box), it just cannot carry a buddy page. The + * chunk allocator below is unaffected, so this is not fatal. */ + LM_INFO("%s: arena of %lu bytes holds no whole %lu byte page - " + "buddy reclaim will be inactive\n", + hb->name, hb->hsize, hps); + return 0; + } + + /* The grid must not describe memory the reservation does not own. Also + * caught by the harness: an npages one too large keeps every internal + * relation intact and only shows up against the reservation end. */ + if (hb->pbase + (hb->npages << shift) > end) { + LM_ERR("%s: page grid of %lu pages ends at %p, past the %lu byte " + "reservation ending at %p\n", hb->name, hb->npages, + hb->pbase + (hb->npages << shift), hb->hsize, end); + return -1; + } + + /* --- the proof. Real addresses, spanning the whole grid. --- */ + for (i = 0; i < 5; i++) { + unsigned long pg, leaf, want_pg; + const char *probe, *pbase_of; + + switch (i) { + case 0: want_pg = 0; off = 0; break; + case 1: want_pg = 0; off = HG_LEAF_SIZE; break; + case 2: want_pg = hb->npages / 2; off = hps / 2; break; + case 3: want_pg = hb->npages - 1; off = 0; break; + default:want_pg = hb->npages - 1; off = hps - 1; break; + } + probe = hb->pbase + (want_pg << shift) + off; + + if (!hg_in_pages(hb, probe)) { + LM_ERR("%s: addressing self-test %d: %p should be inside the " + "%lu-page grid at %p but is not\n", + hb->name, i, probe, hb->npages, hb->pbase); + return -1; + } + pg = hg_page_of(hb, probe); + if (pg != want_pg) { + LM_ERR("%s: addressing self-test %d: %p resolved to page %lu, " + "expected %lu\n", hb->name, i, probe, pg, want_pg); + return -1; + } + pbase_of = hg_page_base(hb, probe); + if ((unsigned long)pbase_of & (hps - 1)) { + LM_ERR("%s: addressing self-test %d: page base %p is not " + "%lu-aligned\n", hb->name, i, pbase_of, hps); + return -1; + } + if (probe - pbase_of != (long)off) { + LM_ERR("%s: addressing self-test %d: %p is %ld bytes into its " + "page, expected %lu\n", hb->name, i, probe, + (long)(probe - pbase_of), off); + return -1; + } + leaf = hg_leaf_of(hb, probe); + if (leaf != off >> HG_LEAF_SHIFT || + leaf >= hg_leaves_per_page(hb)) { + LM_ERR("%s: addressing self-test %d: %p is leaf %lu, expected " + "%lu of %lu\n", hb->name, i, probe, leaf, + off >> HG_LEAF_SHIFT, hg_leaves_per_page(hb)); + return -1; + } + } + + /* and the negative side - the header region and the byte past the end + * must NOT classify as page memory, or a stray pointer would be + * "resolved" to a block that does not exist */ + if (hb->pbase != hb->hbase && hg_in_pages(hb, hb->hbase)) { + LM_ERR("%s: addressing self-test: unaligned head %p classifies as " + "page memory\n", hb->name, hb->hbase); + return -1; + } + if (hg_in_pages(hb, hb->pbase + (hb->npages << shift))) { + LM_ERR("%s: addressing self-test: the byte past the last page " + "classifies as page memory\n", hb->name); + return -1; + } + + LM_DBG("%s page grid: %lu pages of %lu B at %p (shift %u), %lu leaves " + "of %lu B per page; addressing verified\n", + hb->name, hb->npages, hps, hb->pbase, shift, + hg_leaves_per_page(hb), HG_LEAF_SIZE); + return 0; +} + +int hg_arena_init(struct hg_block *hb, unsigned long hdr_size) +{ + unsigned int idx, c, needed; + + if (pages_init(hb) < 0) + return -1; + + /* + * Leave the block header itself untouched by the bump allocator, and + * start on a leaf boundary: a buddy block must be naturally aligned to + * its own size, and every size is a multiple of the leaf, so aligning + * the very first carve is what makes all of them aligned. Costs one + * rounding here - under one leaf, once per arena - and is the whole + * reason the address mask above can be a mask at all. + */ + hb->hoff = (hdr_size + HG_LEAF_SIZE - 1) & ~(HG_LEAF_SIZE - 1); + if (hb->pbase != hb->hbase) { + /* the grid does not start at hbase, so the first carve must clear + * the discarded head too */ + unsigned long skip = (unsigned long)(hb->pbase - hb->hbase); + + if (hb->hoff < skip) + hb->hoff = skip; + } + + /* see chunk_size_for(): no single chunk may swallow a big slice of a + * small arena. /64 keeps all 21 classes plus the large tier inside a + * third of the arena even in the worst case. */ + hb->chunk_max = hb->size / 64; + if (hb->chunk_max > HG_CHUNK_SMALL) + hb->chunk_max = HG_CHUNK_SMALL; + if (hb->chunk_max < HG_CHUNK_MIN) + hb->chunk_max = HG_CHUNK_MIN; + + private_caps_init(hb); + + /* size -> class LUT: needed = requested payload + hidden header */ + for (idx = 0; idx <= HG_CELL_MAX / HG_ROUNDTO; idx++) { + needed = idx * HG_ROUNDTO + HG_CELL_HDR; + for (c = 0; c < HG_NCLASSES; c++) + if (cell_sizes[c] >= needed) + break; + hb->size2class[idx] = (unsigned char)c; /* HG_NCLASSES = oversize */ + } + + LM_DBG("%s arena ready: %d classes, %u B to %u B cells (header=%zu B)\n", + hb->name, HG_NCLASSES, cell_sizes[0], cell_sizes[HG_NCLASSES-1], + HG_CELL_HDR); + + /* + * Last, because it carves its metadata with the bump allocator and then + * reserves everything below the resulting hoff. Anything that bump-carves + * after this point would be handing out memory the buddy believes is + * free, which is why hg_chunk_backing() refuses once buddy_ready is set. + */ + if (hg_buddy_init(hb) < 0) + return -1; + return 0; +} + +/* + * NOTE: this has no callers anywhere in the tree - checked across all of + * modules/ and the core - and is kept only as a published arena API. It is + * routed through the buddy rather than the bump allocator regardless: left on + * the bump it would be a loaded gun, silently double-allocating the moment + * anyone did start calling it (see the guard in hg_chunk_backing()). + */ +void *hg_region_alloc(struct hg_block *hb, unsigned long size) +{ + struct hg_region *rg; + unsigned long need = size + sizeof(struct hg_region) + 64; + char *aligned; + int ord; + + lock_get(&hb->lock); + ord = hg_buddy_order_for(hb, need); + rg = ord < 0 ? NULL : hg_buddy_alloc(hb, (unsigned int)ord); + if (!rg && ord < 0) + rg = hg_buddy_alloc_run(hb, + (need + hb->hps - 1) >> hb->hps_shift); + /* v3: grow-and-retry, same single-retry contract as carve_chunk() */ + if (!rg && hg_buddy_grow(hb, need) == 0) { + rg = ord < 0 + ? hg_buddy_alloc_run(hb, + (need + hb->hps - 1) >> hb->hps_shift) + : hg_buddy_alloc(hb, (unsigned int)ord); + } + lock_release(&hb->lock); + + if (!rg) { + LM_ERR("%s: no more HG_MALLOC arena memory for a %lu byte " + "region\n", hb->name, need); + return NULL; + } + if (ord >= 0) + need = HG_LEAF_SIZE << ord; + rg->size = need; + aligned = (char *)(((unsigned long)rg + sizeof(struct hg_region) + 63) + & ~63UL); + + lock_get(&hb->lock); + rg->next = hb->regions; + hb->regions = rg; + hb->real_used += need; + if (hb->real_used > hb->max_real_used) + hb->max_real_used = hb->real_used; + if ((unsigned long)rg < hb->lo) + hb->lo = (unsigned long)rg; + if ((unsigned long)rg + need > hb->hi) + hb->hi = (unsigned long)rg + need; + lock_release(&hb->lock); + + return aligned; +} + +void hg_arena_destroy(struct hg_block *hb) +{ + int i; + + /* every chunk/region is carved from ONE mmap (hb->hbase) - munmap'd + * as a whole by hg_malloc_destroy() in hg_malloc.c, never individually + * freed here. Just drop the per-process palloc slot, if any. */ + for (i = 0; i < HG_MAX_INSTANCES; i++) + if (palloc_slots[i].owner == hb) + memset(&palloc_slots[i], 0, sizeof(struct hg_palloc)); + hg_last = NULL; + + hg_large_destroy(hb); +} + +void hg_arena_child_init(struct hg_block *hb) +{ + int i; + + /* + * After fork every child holds a COW copy of the parent's private + * allocator state for @hb - the SAME bump pointer and the SAME + * free-list cell addresses. A child must not keep them (two processes + * bumping one chunk would hand out the same cell) and must NOT donate + * them to the global pool either: every child inherited the identical + * copy, so each would push the same physical cells onto the shared + * free list, corrupting it (a cell landing on the list N times, later + * popped and written by several processes at once). + * + * Ported reasoning from cachedb_perf's pcache_arena_child_init(), + * which hit exactly this as the CP-16 corruption bug. The leftover + * cells belong to the parent; the child discards its inherited copy + * and starts empty, carving its own chunk on first use. + * + * Since palloc_slots became __thread this clears only the CALLING + * thread's slots, which is exactly right: fork() clones just the + * calling thread, so the child starts life single-threaded and there + * are no other slots in it to clear. Any IO threads it later spawns + * get freshly-zeroed TLS of their own. + */ + for (i = 0; i < HG_MAX_INSTANCES; i++) + if (palloc_slots[i].owner == hb) + memset(&palloc_slots[i], 0, sizeof(struct hg_palloc)); + hg_last = NULL; +} + +unsigned int hg_cell_total_size(unsigned char cls) +{ + if (cls >= HG_NCLASSES) + return 0; + return cell_sizes[cls]; +} + +#ifdef DBG_MALLOC +void *hg_cell_alloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line) +#else +void *hg_cell_alloc(struct hg_block *hb, unsigned long size) +#endif +{ + struct hg_palloc *pl; + char *cell_start, *payload; + unsigned int got; + int c; + + if (size + HG_CELL_HDR > HG_CELL_MAX) { +#ifdef DBG_MALLOC + return hg_large_alloc(hb, size, file, func, line); +#else + return hg_large_alloc(hb, size); +#endif + } + c = hb->size2class[(size + HG_ROUNDTO - 1) / HG_ROUNDTO]; + if (c >= HG_NCLASSES) + return NULL; + + pl = hg_get_palloc(hb); + if (!pl) + return NULL; + + /* fast paths: no locks, no shared lines */ + cell_start = pl->cls[c].free_head; + if (cell_start) { + pl->cls[c].free_head = cell_next(cell_start); + pl->cls[c].nfree--; + goto found; + } + if (pl->cls[c].left) { + cell_start = pl->cls[c].bump; + pl->cls[c].bump += cell_sizes[c]; + pl->cls[c].left--; + goto found; + } + + /* slow path: refill from the global pool, else carve a chunk */ + lock_get(&hb->lock); + for (got = 0; got < HG_REFILL_BATCH; got++) { + cell_start = gpool_pop(hb, c); + if (!cell_start) + break; + cell_set_next(cell_start, pl->cls[c].free_head); + pl->cls[c].free_head = cell_start; + pl->cls[c].nfree++; + } + if (!got) { + /* + * About to grow the arena. Reserve pressure IS the trigger the + * design asks for ("trigger the sweep on reserve pressure, not + * only on time"), and here the owning thread is the one asking - + * so it flushes its own cache inline, with no IPC and no + * self-addressed RPC to order against. + * + * Often this alone satisfies the request: the cells were ours all + * along, just invisible. When it does not, it has at least made + * the block accounting true, so blocks that were already empty + * can be reclaimed instead of the arena growing around them. + */ + unsigned int flushed = cache_flush_locked(hb, pl); + + if (flushed) { + for (got = 0; got < HG_REFILL_BATCH; got++) { + cell_start = gpool_pop(hb, c); + if (!cell_start) + break; + cell_set_next(cell_start, pl->cls[c].free_head); + pl->cls[c].free_head = cell_start; + pl->cls[c].nfree++; + } + LM_DBG("%s class %d: flushed %u cached cells under pressure, " + "recovered %d\n", hb->name, c, flushed, got); + } + } + if (!got && carve_chunk(hb, c, pl) < 0) { + lock_release(&hb->lock); + return NULL; + } + lock_release(&hb->lock); + + cell_start = pl->cls[c].free_head; + if (cell_start) { + pl->cls[c].free_head = cell_next(cell_start); + pl->cls[c].nfree--; + goto found; + } + cell_start = pl->cls[c].bump; + pl->cls[c].bump += cell_sizes[c]; + pl->cls[c].left--; + +found: + payload = cell_start + HG_CELL_HDR; +#ifdef DBG_MALLOC + { + const char **hfile = (const char **)(cell_start + HG_ROUNDTO); + const char **hfunc = (const char **)(cell_start + HG_ROUNDTO * 2); + unsigned long *hline = (unsigned long *)(cell_start + HG_ROUNDTO * 3); + *hfile = file; + *hfunc = func; + *hline = line; + } +#endif + /* per-process slot: no lock, no shared cache line (see hg_pstat) */ + { + struct hg_pstat *ps = hg_pstat_mine(hb); + ps->used += cell_sizes[c] - HG_CELL_HDR; + ps->fragments++; + ps->cell_live += cell_sizes[c]; + } + return payload; +} + +#ifdef DBG_MALLOC +void hg_cell_free(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line) +#else +void hg_cell_free(struct hg_block *hb, void *p) +#endif +{ + struct hg_palloc *pl; + char *cell_start; + unsigned int c, i; + void *d; + + if (!p) + return; + + cell_start = HG_HDR(p); + + /* before the class byte is even read: an out-of-arena pointer may + * not be mapped at all, and reading it would fault here rather + * than merely corrupt a pool. See hg_owns(). */ + if (!hg_owns(hb, cell_start)) { + struct hg_block *owner = hg_owner(cell_start); + + if (!owner) { + hg_corrupt(hb, HG_C_FOREIGN_PTR); + LM_CRIT("%s: %p belongs to no live arena - refusing to " + "free it\n", hb->name, p); + return; + } + + /* + * A different arena of ours issued this cell, which is normal + * after fork: the child runs on its own pkg arena while + * modules still release things the parent allocated pre-fork. + * Hand it back to whoever owns it. Straight to that arena's + * global pool, deliberately - the per-thread cache exists to + * speed up reuse, and this process will never allocate from + * the foreign arena again, so caching there would strand the + * cell instead of freeing it. + */ + hg_xarena_frees++; + hg_cell_free_global(owner, p); + return; + } + + c = *(unsigned char *)cell_start; + + if (c == HG_LARGE_MARKER) { + hg_large_free(hb, (struct hg_lfrag *)(void *)(cell_start - HG_LFRAG_HDR)); + return; + } + + if (c >= HG_NCLASSES) { + /* the class byte was clobbered - freeing through it would + * corrupt the pools; leak the cell and shout instead. This is + * the only corruption check Phase 1 has - true double-free + * detection (has THIS pointer already been freed) needs a + * per-cell allocated/free bit that isn't implemented yet, see + * the file header note on Phase 1 scope. */ + hg_corrupt(hb, HG_C_BAD_CLASS); + LM_CRIT("%s: cell %p carries invalid class %u - leaking it\n", + hb->name, p, c); + return; + } + + pl = hg_get_palloc(hb); + if (!pl) { + hg_cell_free_global(hb, p); + return; + } + + { + struct hg_pstat *ps = hg_pstat_mine(hb); + ps->used -= cell_sizes[c] - HG_CELL_HDR; + ps->fragments--; + ps->cell_live -= cell_sizes[c]; + } + + cell_set_next(cell_start, pl->cls[c].free_head); + pl->cls[c].free_head = cell_start; + pl->cls[c].nfree++; + + /* keep hoarding bounded: donate half once over this class's threshold. + * The threshold is a cell count derived from a byte budget, so a big + * class trips it after very few cells - and a class whose priv_max is 0 + * trips it on the first free, which is exactly the intent: never cache + * that class privately. */ + if (pl->cls[c].nfree > hb->priv_max[c]) { + unsigned int claimed = pl->cls[c].nfree; + unsigned int donate = hb->priv_donate[c]; + + lock_get(&hb->lock); + for (i = 0; i < donate; i++) { + d = pl->cls[c].free_head; + /* priv_donate is half priv_max, so a chain whose + * length matches nfree can always satisfy this loop. + * Reaching a NULL here therefore means nfree has + * drifted ABOVE the number of cells actually on the + * chain, and walking on would evaluate cell_next(NULL) + * - the production SIGSEGV this guard replaces (a TCP + * worker died with cls[5] = {free_head = NULL, + * nfree = 257} and si_addr = HG_CELL_HDR). */ + if (!d) + break; + pl->cls[c].free_head = cell_next(d); + pl->cls[c].nfree--; + gpool_push(hb, c, d); + } + /* The chain ran dry, so the truth is zero. Resyncing is not + * cosmetic: leaving nfree high would re-enter this block on + * the very next free of this class and trip the guard again + * on every single call, burying the log. */ + if (i < donate) + pl->cls[c].nfree = 0; + lock_release(&hb->lock); + + if (i < donate) { + hg_corrupt(hb, HG_C_NFREE_UNDERFLOW); + LM_CRIT("%s: class %u free list ran dry after %u cells " + "but nfree claimed %u - counter resynced to 0, " + "pool accounting drifted\n", + hb->name, c, i, claimed); + } + } +} + +void hg_cell_free_global(struct hg_block *hb, void *p) +{ + char *cell_start; + unsigned int c; + + if (!p) + return; + + cell_start = HG_HDR(p); + if (!hg_owns(hb, cell_start)) { + struct hg_block *owner = hg_owner(cell_start); + + /* see the matching comment in hg_cell_free() - reached both + * directly and by that function's redirect, so the redirect + * must not loop: it only ever passes the resolved owner. */ + if (!owner) { + hg_corrupt(hb, HG_C_FOREIGN_PTR); + LM_CRIT("%s: %p belongs to no live arena - refusing to " + "free it\n", hb->name, p); + return; + } + if (owner != hb) { + hg_xarena_frees++; + hb = owner; + } + } + c = *(unsigned char *)cell_start; + if (c == HG_LARGE_MARKER) { + hg_large_free(hb, (struct hg_lfrag *)(void *)(cell_start - HG_LFRAG_HDR)); + return; + } + if (c >= HG_NCLASSES) { + hg_corrupt(hb, HG_C_BAD_CLASS); + LM_CRIT("%s: cell %p carries invalid class %u - leaking it\n", + hb->name, p, c); + return; + } + /* hg_cell_free() delegates here WITHOUT having touched the counters + * (it returns early when this process has no palloc), and this is that + * function's only caller - so the decrement belongs here, not there. + * Before this, a free taken down this path left used/fragments + * permanently over-reported. */ + { + struct hg_pstat *ps = hg_pstat_mine(hb); + ps->used -= cell_sizes[c] - HG_CELL_HDR; + ps->fragments--; + ps->cell_live -= cell_sizes[c]; + } + + lock_get(&hb->lock); + gpool_push(hb, c, cell_start); + lock_release(&hb->lock); +} + +/* + * Cell-slot bytes that are carved into chunks but NOT currently handed out: + * cells on a private free stack, cells in the global pool, and cells never + * bumped at all. This capacity is fully reusable. + * + * It exists because HG_MALLOC never un-carves a chunk, so hb->real_used (the + * arena's own footprint) can only ever rise. q_malloc/f_malloc report a freed + * fragment sitting on a free list as FREE; without subtracting this figure, + * HG_MALLOC would report the identical state as permanently USED, and + * real_used would read as an unbounded leak on any monitoring dashboard. + * + * Walks the chunk registry WITHOUT hb->lock, deliberately: the list is + * append-only, ch->cells/ch->cell_size are immutable once carved, and the + * head pointer is published after the chunk is fully built, so a reader + * either sees a complete chunk or does not see it at all. Taking the lock + * here would nest it under whatever the stats caller already holds, for a + * figure that is a sample either way. + */ +unsigned long hg_slab_recycled(struct hg_block *hb) +{ + struct hg_chunk *ch; + unsigned long capacity = 0, live; + + /* + * Under the lock, which it did not need while chunks were immortal. + * gc_class() now unlinks a chunk and immediately hands the block to + * hg_buddy_free(), whose fl_push() overwrites the first 24 bytes - + * next, prev, cls, cell_size. A reader walking this list lock-free + * (every SHM_GET_RUSED, every stats scrape, from any process at any + * time) would follow a ch->next that is now a free-list pointer or a + * magic value. Reading stats must not be able to walk into a block the + * allocator has already recycled. + */ + lock_get(&hb->lock); + for (ch = hb->chunks; ch; ch = ch->next) + capacity += (unsigned long)ch->cells * ch->cell_size; + lock_release(&hb->lock); + + live = hg_cell_live(hb); + return capacity > live ? capacity - live : 0; +} + +void hg_arena_stats(struct hg_block *hb, unsigned int *nchunks, + unsigned long *bytes) +{ + lock_get(&hb->lock); + *nchunks = hb->nchunks; + *bytes = hb->real_used; + lock_release(&hb->lock); +} + +/* + * Live-cell enumeration (mem-group stats_core_init, hg_status_dbg's dump). + * + * Small cells (unlike large frags' f->prev) carry no per-cell free/live + * flag - a cell's free-ness is only implicit via free-list membership. So + * "walk every live cell" first collects every currently-free address into + * a plain glibc-malloc'd hash set (deliberately NOT pkg/shm-backed - this + * is diagnostic machinery examining the allocator itself, matching + * mem_dbg_hash.c's own independence), then walks every chunk's cells + * checking membership. + * + * Accuracy: exact when called pre-fork (hg_stats_core_init(), single + * process, no siblings yet to hide cells in their own private free + * stacks). Best-effort post-fork (hg_status_dbg()'s memdump path): this + * process's own private free stack IS included, but a cell idling in some + * OTHER worker's private free_head chain at the moment of the walk will be + * misreported as live - an inherent consequence of the lock-free private- + * cache design, not a bug. Documented, not silently pretended away. + */ +/* + * The set of currently-free cells, for the live-cell walker. + * + * OPEN ADDRESSED, in ONE allocation. It used to be bucket chains with a + * malloc() per free cell, and that was not merely slow: hg_free_set_add() + * silently gave up when an entry allocation failed, the walker infers + * liveness by ABSENCE from this set, and so a dropped insert handed a FREE + * cell to the callback as live. Under DBG_MALLOC the callback then reads + * file/func out of that cell's header - which, for a free cell, holds the + * free-list link - and consumes arena pointers as strings. The comment there + * called it "never a correctness issue"; it was an abort, and it triggered + * under memory pressure, which is exactly when someone takes a memory dump. + * + * One allocation removes that entirely: there is no per-cell allocation left + * to fail mid-walk, and nothing is malloc'd while hb->lock is held except the + * single table. If that one allocation fails the walk is skipped cleanly, + * which is a refusal rather than a corruption. + * + * Linear probing at a load factor of 0.5, so a probe always terminates on an + * empty slot, and the whole table is one contiguous array of pointers - + * better locality than chasing chains, on a walk that touches every cell. + */ +struct hg_free_set { + void **slots; /* NULL = empty; power-of-two count */ + unsigned int nslots; + unsigned int nused; + unsigned int overflow; /* inserts refused - must stay 0, see below */ +}; + +static inline unsigned int hg_ptr_hash(const void *p, unsigned int nbuckets) +{ + unsigned long v = (unsigned long)p; + + v ^= v >> 16; + v *= 2654435761UL; /* Knuth multiplicative hash */ + v ^= v >> 13; + return (unsigned int)(v & (nbuckets - 1)); +} + +static int hg_free_set_init(struct hg_free_set *set, unsigned int expected) +{ + unsigned int n = 64; + + while (n < expected * 2 && n < (1U << 24)) + n <<= 1; + set->slots = calloc(n, sizeof(*set->slots)); + if (!set->slots) + return -1; + set->nslots = n; + set->nused = 0; + set->overflow = 0; + return 0; +} + +static void hg_free_set_add(struct hg_free_set *set, void *addr) +{ + unsigned int h = hg_ptr_hash(addr, set->nslots); + + /* + * Refuse rather than wrap forever. The count was taken under the same + * hb->lock that is still held, so the table cannot be undersized unless + * a free list is longer than its own counter claims - which is the + * nfree-drift the allocator already detects elsewhere. Counting it lets + * the caller say the set is incomplete instead of the walker quietly + * reporting free cells as live. + */ + if (set->nused * 2 >= set->nslots) { + set->overflow++; + return; + } + while (set->slots[h]) { + if (set->slots[h] == addr) + return; + h = (h + 1) & (set->nslots - 1); + } + set->slots[h] = addr; + set->nused++; +} + +static int hg_free_set_has(struct hg_free_set *set, void *addr) +{ + unsigned int h = hg_ptr_hash(addr, set->nslots); + + while (set->slots[h]) { + if (set->slots[h] == addr) + return 1; + h = (h + 1) & (set->nslots - 1); + } + return 0; +} + +static void hg_free_set_destroy(struct hg_free_set *set) +{ + free(set->slots); + set->slots = NULL; +} + +/* exact number of cells on every free list this process can see, counted + * under hb->lock so the table can be sized once and never grown */ +static unsigned int hg_free_set_count(struct hg_block *hb) +{ + struct hg_palloc *pl = hg_get_palloc(hb); + struct hg_chunk *ch; + unsigned int n = 0; + void *cur; + int c; + + for (ch = hb->chunks; ch; ch = ch->next) + for (cur = ch->free_head; cur; cur = cell_next(cur)) + n++; + if (pl) + for (c = 0; c < HG_NCLASSES; c++) + for (cur = pl->cls[c].free_head; cur; cur = cell_next(cur)) + n++; + return n; +} + +static void hg_free_set_populate(struct hg_block *hb, struct hg_free_set *set) +{ + int c; + void *cur; + struct hg_chunk *ch; + struct hg_palloc *pl = hg_get_palloc(hb); + + /* the shared free cells are per BLOCK now, so walk the registry rather + * than one list per class - hb->chunks reaches every live block */ + for (ch = hb->chunks; ch; ch = ch->next) + for (cur = ch->free_head; cur; cur = cell_next(cur)) + hg_free_set_add(set, cur); + + for (c = 0; c < HG_NCLASSES; c++) { + if (pl) + for (cur = pl->cls[c].free_head; cur; cur = cell_next(cur)) + hg_free_set_add(set, cur); + } +} + +void hg_arena_walk_live(struct hg_block *hb, + void (*cb)(void *payload, void *ctx), void *ctx) +{ + struct hg_free_set set; + struct hg_chunk *ch; + unsigned int i, total_free = 0; + char *cell_start; + + lock_get(&hb->lock); + + /* + * Count exactly rather than estimating from gpool_n[]: that counts only + * the SHARED pool, while the set must also hold this thread's private + * cache, so the estimate was low and the table was grown by a malloc per + * cell to cover the difference. Both walks run under the same hb->lock, + * so the count cannot go stale between counting and filling. + */ + total_free = hg_free_set_count(hb); + + if (hg_free_set_init(&set, total_free) < 0) { + lock_release(&hb->lock); + LM_ERR("%s: out of memory building the live-cell diagnostic " + "set - skipping the walk\n", hb->name); + return; + } + hg_free_set_populate(hb, &set); + + /* + * Must not happen: the table was sized from a count taken under this + * same lock. If it does, a free list is longer than its own counter + * says, and the walk would report free cells as live - so say so rather + * than emit a quietly wrong dump. + */ + if (set.overflow) + LM_CRIT("%s: live-cell set overflowed by %u - the dump below " + "may report free cells as live\n", hb->name, set.overflow); + + for (ch = hb->chunks; ch; ch = ch->next) { + for (i = 0; i < ch->cells; i++) { + cell_start = (char *)ch + sizeof(struct hg_chunk) + + (unsigned long)i * ch->cell_size; + if (!hg_free_set_has(&set, cell_start)) + cb(cell_start + HG_CELL_HDR, ctx); + } + } + + hg_free_set_destroy(&set); + lock_release(&hb->lock); +} + +#ifdef SHM_EXTRA_STATS +static void hg_stats_core_init_cb(void *payload, void *ctx) +{ + HG_STATS_IDX(payload) = (unsigned long)*(int *)ctx; +} + +void hg_arena_stats_core_init(struct hg_block *hb, int core_index) +{ + hg_arena_walk_live(hb, hg_stats_core_init_cb, &core_index); + hg_large_stats_core_init(hb, core_index); +} +#endif + + + +/* ========================================================================= + * Statistics (shm arena only - see hg_register_stats() in hg_malloc.h) + * + * The same figures hg_stats reports over MI, as core statistics so a dashboard + * can graph them. Without this, hugepage consumption shows up only as the + * host pool draining (node_exporter's HugePages_Free) with nothing attributing + * it to the allocator. + * + * NAMING IS DELIBERATE: hg_shm_* under the "hgmem" module, kept clearly apart + * from cachedb_perf's hugepage_arena_* statistics. Those describe that + * module's OWN optional dedicated reservation and read 0 when it has none - + * different memory entirely. The two have already been confused once: a + * cachedb_perf arena reported as "plain 4K pages" was read as the cache + * sitting on small pages when its data was really in HG_MALLOC's 2M-backed + * shm. Anything ambiguous here invites that mistake again, or double-counting. + * ========================================================================= */ + +#include "../statistics.h" +#include "shm_mem.h" /* mem_allocator_shm */ + +enum hg_stat_field { + HGS_TIER = 0, HGS_TOTAL, HGS_PINNED_BYTES, HGS_CARVED, HGS_CARVED_PEAK, + HGS_CHUNKS, HGS_FREE_TO_CARVE, HGS_LIVE, HGS_LIVE_PEAK, HGS_PAYLOAD, + HGS_CELLS, HGS_SLAB_LIVE, HGS_SLAB_RECYCLED, + /* v2 reclaim rates. Counters, not events: at 800 CPS the allocator sees + * ~10^5 cell ops/s and cache/block transitions are 1-3% of that, so an + * EVI event here would cost more than the allocator it reports on. */ + HGS_BLOCKS_CARVED, HGS_BLOCKS_RETURNED, HGS_GC_PASSES, + HGS_CACHE_FLUSHES, HGS_CELLS_FLUSHED, + HGS_BUDDY_SPLITS, HGS_BUDDY_MERGES, HGS_BUDDY_FREE_LEAVES, + /* large tier footprint, and the reserve floor - without these last + * three the floor is invisible outside a one-shot log line */ + HGS_LARGE_BACKING, HGS_LARGE_LIVE, HGS_LARGE_RECYCLED, + HGS_LARGE_CHUNKS_CARVED, HGS_LARGE_CHUNKS_RETURNED, + HGS_RESERVE_FLOOR, HGS_BELOW_FLOOR, HGS_FLOOR_CROSSINGS, + /* one number an operator can alert on; the breakdown is in hg_stats */ + HGS_CORRUPTION, + /* v3 elastic arena: committed vs reserved, and the grow ledger. + * grow_refused is the alertable one - a nonzero, rising value means + * demand hit a wall (cap or host), which is exactly the old + * exhaustion condition wearing its new name. */ + HGS_COMMITTED, HGS_CAP, HGS_GROWS, HGS_GROW_BYTES, HGS_GROW_REFUSED, + HGS_SHRINKS, HGS_SHRINK_BYTES, + /* the alertable gauge: 1 while a RESOURCE refusal is latched (cap + * refusals never latch - an admin ceiling is policy, not incident) */ + HGS_GROW_BLOCKED, +}; + +static unsigned long hg_shm_stat(void *ctx) +{ + struct hg_block *hb = (struct hg_block *)shm_block; + + if (!hb) + return 0; + switch ((enum hg_stat_field)(long)ctx) { + case HGS_TIER: return hb->tier; + case HGS_TOTAL: return hb->size; + /* bytes rather than the MB the MI reports, so it composes with the + * other byte-valued statistics and with node_exporter's page counts */ + case HGS_PINNED_BYTES: return (unsigned long)hb->locked_mb << 20; + /* carved: taken from the arena - slab blocks plus whole large-tier + * chunks - and what free_to_carve counts down from. It is NOT + * monotonic in v2: gc_class() returns a drained block to the buddy and + * subtracts it here, so carved falling below carved_peak is the normal, + * intended signal that reclaim is working. */ + case HGS_CARVED: return hb->real_used; + case HGS_CARVED_PEAK: return hb->max_real_used; + case HGS_CHUNKS: return hb->nchunks; + case HGS_FREE_TO_CARVE: return hb->size - hb->real_used; + /* live: handed out right now. NOT the same as carved - the gap is + * memory sitting on per-process free stacks, reusable only within its + * own size class, which is expected slab behaviour and not a leak. */ + case HGS_LIVE: return hg_get_real_used(hb); + case HGS_LIVE_PEAK: return hb->max_live_used; + case HGS_PAYLOAD: return hg_used(hb); + case HGS_CELLS: return hg_fragments(hb); + /* SLAB ONLY: large allocations go to the boundary-tag tier and never + * appear here, so this is deliberately NOT comparable with payload */ + case HGS_SLAB_LIVE: return hg_cell_live(hb); + case HGS_SLAB_RECYCLED: return hg_slab_recycled(hb); + /* Reclaim rates. blocks_carved counts every block ever cut and + * blocks_returned every one handed back, so carved-minus-returned is + * the live block count and the RATIO is how well reclaim is keeping + * up - which is the figure the whole v2 rework is judged on, and the + * one that must be readable from MI rather than grepped out of a debug + * log under load. */ + case HGS_BLOCKS_CARVED: return hb->blocks_carved; + case HGS_BLOCKS_RETURNED: return hb->gc_blocks_returned; + case HGS_GC_PASSES: return hb->gc_passes; + case HGS_CACHE_FLUSHES: return hb->cache_flushes; + case HGS_CELLS_FLUSHED: return hb->cells_flushed; + case HGS_BUDDY_SPLITS: return hb->buddy_splits; + case HGS_BUDDY_MERGES: return hb->buddy_merges; + case HGS_BUDDY_FREE_LEAVES: return hb->buddy_free_leaves; + case HGS_LARGE_BACKING: return hb->large_backing; + case HGS_LARGE_LIVE: return hb->large_live; + case HGS_LARGE_RECYCLED: return hg_large_recycled(hb); + case HGS_LARGE_CHUNKS_CARVED: return hb->large_chunks_carved; + case HGS_LARGE_CHUNKS_RETURNED: return hb->large_chunks_returned; + case HGS_RESERVE_FLOOR: return hb->reserve_floor; + case HGS_BELOW_FLOOR: return hb->below_floor; + case HGS_FLOOR_CROSSINGS: return hb->floor_crossings; + case HGS_CORRUPTION: return hg_corrupt_total(hb); + case HGS_COMMITTED: return hb->hsize; + case HGS_CAP: return hb->hcap; + case HGS_GROWS: return hb->grows; + case HGS_GROW_BYTES: return hb->grow_bytes; + case HGS_GROW_REFUSED: return hb->grow_refused; + case HGS_GROW_BLOCKED: return hb->grow_blocked; + case HGS_SHRINKS: return hb->shrinks; + case HGS_SHRINK_BYTES: return hb->shrink_bytes; + } + return 0; +} + +static const struct { + const char *name; + enum hg_stat_field field; +} hg_stat_defs[] = { + {"hg_shm_tier", HGS_TIER}, + {"hg_shm_total_size", HGS_TOTAL}, + {"hg_shm_pinned_bytes", HGS_PINNED_BYTES}, + {"hg_shm_carved", HGS_CARVED}, + {"hg_shm_carved_peak", HGS_CARVED_PEAK}, + {"hg_shm_chunks", HGS_CHUNKS}, + {"hg_shm_free_to_carve", HGS_FREE_TO_CARVE}, + {"hg_shm_live", HGS_LIVE}, + {"hg_shm_live_peak", HGS_LIVE_PEAK}, + {"hg_shm_live_payload", HGS_PAYLOAD}, + {"hg_shm_live_cells", HGS_CELLS}, + {"hg_shm_slab_live", HGS_SLAB_LIVE}, + {"hg_shm_slab_recycled", HGS_SLAB_RECYCLED}, + {"hg_shm_blocks_carved", HGS_BLOCKS_CARVED}, + {"hg_shm_blocks_returned", HGS_BLOCKS_RETURNED}, + {"hg_shm_gc_passes", HGS_GC_PASSES}, + {"hg_shm_cache_flushes", HGS_CACHE_FLUSHES}, + {"hg_shm_cells_flushed", HGS_CELLS_FLUSHED}, + {"hg_shm_buddy_splits", HGS_BUDDY_SPLITS}, + {"hg_shm_buddy_merges", HGS_BUDDY_MERGES}, + {"hg_shm_buddy_free_leaves", HGS_BUDDY_FREE_LEAVES}, + {"hg_shm_large_backing", HGS_LARGE_BACKING}, + {"hg_shm_large_live", HGS_LARGE_LIVE}, + {"hg_shm_large_recycled", HGS_LARGE_RECYCLED}, + {"hg_shm_large_chunks_carved", HGS_LARGE_CHUNKS_CARVED}, + {"hg_shm_large_chunks_returned", HGS_LARGE_CHUNKS_RETURNED}, + {"hg_shm_reserve_floor", HGS_RESERVE_FLOOR}, + {"hg_shm_below_floor", HGS_BELOW_FLOOR}, + {"hg_shm_floor_crossings", HGS_FLOOR_CROSSINGS}, + {"hg_shm_corruption", HGS_CORRUPTION}, + {"hg_shm_committed", HGS_COMMITTED}, + {"hg_shm_cap", HGS_CAP}, + {"hg_shm_grows", HGS_GROWS}, + {"hg_shm_grow_bytes", HGS_GROW_BYTES}, + {"hg_shm_grow_refused", HGS_GROW_REFUSED}, + {"hg_shm_grow_blocked", HGS_GROW_BLOCKED}, + {"hg_shm_shrinks", HGS_SHRINKS}, + {"hg_shm_shrink_bytes", HGS_SHRINK_BYTES}, + {NULL, 0} +}; + +int hg_register_stats(void) +{ + int i; + + if (mem_allocator_shm != MM_HG_MALLOC && + mem_allocator_shm != MM_HG_MALLOC_DBG) + return 0; /* a different allocator is in use - nothing to say */ + + for (i = 0; hg_stat_defs[i].name; i++) { + if (register_stat2("hgmem", (char *)hg_stat_defs[i].name, + (stat_var **)hg_shm_stat, STAT_NO_RESET|STAT_IS_FUNC, + (void *)(long)hg_stat_defs[i].field, 0) != 0) { + LM_ERR("failed to add the %s statistic\n", hg_stat_defs[i].name); + return -1; + } + } + return 0; +} + +#endif /* HG_MALLOC - covers the statistics section below the arena + * code too. It used to close at what is now ~line 1690, leaving + * hg_shm_stat() and hg_register_stats() - which dereference + * struct hg_block and shm_block - compiled unconditionally. A + * build without -DHG_MALLOC failed there with 22 "invalid use of + * undefined type" errors. Nothing in that section is meaningful + * without the allocator. */ diff --git a/mem/hg_arena.h b/mem/hg_arena.h new file mode 100644 index 00000000000..623b576c7fb --- /dev/null +++ b/mem/hg_arena.h @@ -0,0 +1,144 @@ +/* + * hugepage-backed slab allocator - arena internals + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * Slab arena, adapted from cachedb_perf's pcache_arena.{c,h} (DESIGN 2.6.1 / + * 3.3): cells live in fixed-size chunks bump-carved from a huge-page + * reservation and NEVER returned while the server runs - that is what keeps + * the fast path lock-free. A chunk is permanently bound to one size class; + * cells never straddle or move (Phase 1 has no defrag/GC - see the estimate + * discussion; the only mitigation is idle-chunk return, added in a later + * pass, not this one). + * + * Two structural differences from cachedb_perf, both required because + * HG_MALLOC is a general-purpose shm/pkg allocator, not a single cache + * module's private arena: + * + * 1. cachedb_perf's cells put the class-id byte at the FRONT of the + * pointer it hands to its own (fully-controlled) callers. HG_MALLOC + * hands pointers to arbitrary core/module code via shm_malloc(), so the + * class tag lives in a hidden header BEFORE the payload instead + * (HG_HDR/HG_CLASS in hg_malloc.h) - callers see a normal, aligned + * pointer, exactly like FM_FRAG(p) in f_malloc. + * + * 2. cachedb_perf has exactly one arena instance system-wide (a static + * global). HG_MALLOC can have several live instances in one process at + * once (shm, shm_dbg under DBG_MALLOC, and pkg, if pkg also selects + * HG_MALLOC) so every function here takes an explicit struct hg_block * + * and the per-process private free-stack state is looked up per-block + * (hg_get_palloc()), not a single global struct. + */ + +#ifndef hg_arena_h +#define hg_arena_h + +#include "hg_malloc.h" + +/* builds the size -> class lookup table and positions the bump offset past + * the block header (@hdr_size bytes, already reserved by hg_malloc_init() + * at the front of hb->hbase) - called once, right after that header is + * laid out */ +int hg_arena_init(struct hg_block *hb, unsigned long hdr_size); + +/* bump-carve @size bytes from hb's own hugepage reservation, 64-aligned. + * NULL if exhausted - no fallback to another allocator (design decision). + * Exposed for hg_large.c, which grows by carving additional chunks from + * this SAME underlying arena rather than a separate reservation. */ +void *hg_chunk_backing(struct hg_block *hb, unsigned long size); + +/* Flush this THREAD's cached cells back to the shared pool, in every arena it + * caches in. Must run on the owning thread - the caches are __thread and are + * unreachable from anywhere else, which is the whole reason the sweep is + * dispatched rather than executed centrally. */ +void hg_cache_flush_self(void); + +/* Flush only if a sweep has been requested since this thread last looked. + * For threads IPC cannot reach - TCP main's IO pool waits on a condvar, not + * the reactor - called at a job boundary. Cheap: one read of a global. */ +extern volatile unsigned long hg_sweep_gen; + +/* Evaluate the reserve floor. Called from the buddy layer on every grid + * allocation and free, because that is the one point every consumer passes - + * see the definition in hg_arena.c for what went wrong when it lived in + * carve_chunk() instead. hb->lock must be held. */ +void hg_reserve_floor_check(struct hg_block *hb); + +/* Widen the [lo,hi] extent watermarks over a region just handed out. Called + * from the buddy layer so every consumer of grid space is covered - the DBG + * free guard aborts on pointers outside this range. hb->lock must be held. */ +void hg_extent_note(struct hg_block *hb, void *base, unsigned long size); +void hg_cache_flush_if_due(void); +void *hg_backing_aligned(struct hg_block *hb, unsigned long size, + unsigned long align); + +/* a cell of at least @size usable bytes (header excluded), or NULL if + * size > HG_CELL_MAX or the arena is exhausted. No fallback to another + * allocator on exhaustion - fail loud, per the HG_MALLOC design decision + * (increase -m/-M instead). DBG_MALLOC variant stamps file/func/line into + * the cell header at EVERY call, since a cell is reused across many + * allocations over its life (unlike the class id, stamped once at carve + * time and immutable). */ +#ifdef DBG_MALLOC +void *hg_cell_alloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line); +void hg_cell_free(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line); +#else +void *hg_cell_alloc(struct hg_block *hb, unsigned long size); +void hg_cell_free(struct hg_block *hb, void *p); +#endif + +/* a raw, 64-byte-aligned, never-freed region - same backing seam and + * never-returned guarantee as chunks, but NOT carved into cells and NOT + * zeroed. Not on the hg_malloc()/hg_free() hot path in Phase 1; kept for + * future index-structure consumers (mirrors pcache_region_alloc). */ +void *hg_region_alloc(struct hg_block *hb, unsigned long size); + +/* cross-process free (not used in Phase 1 - no expiry/maintenance worker at + * the allocator level - kept for API symmetry with the arena this was + * ported from, and for the shared-pool donate/refill path itself) */ +void hg_cell_free_global(struct hg_block *hb, void *cell); + +/* re-sync after fork(): see hg_malloc_child_init() in hg_malloc.c */ +void hg_arena_child_init(struct hg_block *hb); + +void hg_arena_destroy(struct hg_block *hb); + +/* hg_slab_recycled() is declared in hg_malloc.h, which this header + * already includes - do not redeclare it here */ +void hg_arena_stats(struct hg_block *hb, unsigned int *nchunks, + unsigned long *bytes); + +/* invokes @cb(payload_ptr, ctx) for every currently-live small cell across + * every chunk. Exact pre-fork, best-effort post-fork (see the file-header + * comment in hg_arena.c above this function's definition for why). Used by + * hg_arena_stats_core_init() and hg_status_dbg(). */ +void hg_arena_walk_live(struct hg_block *hb, + void (*cb)(void *payload, void *ctx), void *ctx); + +#ifdef SHM_EXTRA_STATS +/* tags every currently-live cell/frag (small + large) with @core_index - + * called once, pre-fork, mirroring hp_init_shm_statistics() */ +void hg_arena_stats_core_init(struct hg_block *hb, int core_index); +#endif + +#endif /* hg_arena_h */ diff --git a/mem/hg_buddy.c b/mem/hg_buddy.c new file mode 100644 index 00000000000..4b0fddbfa4a --- /dev/null +++ b/mem/hg_buddy.c @@ -0,0 +1,970 @@ +/* + * buddy allocator over the HG_MALLOC huge-page grid + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef HG_MALLOC + +#include +#include + +#include "hg_version.h" +#include "hg_malloc.h" +#include "hg_buddy.h" +#include "hg_arena.h" +#include "../dprint.h" +#include "../globals.h" + +/* + * Two records describe the same tree, because they answer different + * questions and neither answers both cheaply: + * + * leaforder[leaf] the order of the block CONTAINING that leaf - every leaf + * of a block carries it, not just the first. That is what + * turns any interior address into its block with a mask, + * which the layers above need: a cell being freed sits + * somewhere in the middle of its block, and finding the + * block is how the live count gets decremented at all. + * Filling the range costs a memset of 2^order bytes on the + * slow path (256 B for a whole 2 MB page) and buys an O(1) + * lookup on a path that would otherwise need a search. + * HG_LEAF_NONE means the leaf is not buddy space - it + * belongs to a multi-page run. + * + * bitmap[node] 1 iff that tree node is a WHOLE FREE block, i.e. it is + * sitting in a free list right now. Not "free" in the + * sense of "contains free space": a split node is 0 even + * though both its halves may be free. That is exactly the + * predicate the merge step needs - "is my buddy free AND + * entire" - and it answers it in one bit test. + * + * Keeping both is what makes split and merge O(1) instead of a search. + */ + +/* node id in the per-page tree. Level 0 is the whole page (one node), level + * `top` is the leaves; a complete tree over 2^top leaves has 2^(top+1)-1 + * nodes - 511 for a 2 MB page with 8 KB leaves, which is the 64 byte bitmap + * the design budgets. */ +static inline unsigned long node_id(unsigned int top, unsigned int order, + unsigned long leaf) +{ + unsigned int level = top - order; + + return (1UL << level) - 1 + (leaf >> order); +} + +static inline unsigned long nodes_per_page(unsigned int top) +{ + return (1UL << (top + 1)) - 1; +} + +static inline int bit_test(const unsigned long *bm, unsigned long n) +{ + return (bm[n / (sizeof(long) * 8)] >> (n % (sizeof(long) * 8))) & 1UL; +} + +static inline void bit_set(unsigned long *bm, unsigned long n) +{ + bm[n / (sizeof(long) * 8)] |= 1UL << (n % (sizeof(long) * 8)); +} + +static inline void bit_clear(unsigned long *bm, unsigned long n) +{ + bm[n / (sizeof(long) * 8)] &= ~(1UL << (n % (sizeof(long) * 8))); +} + +/* --- free lists ------------------------------------------------------- */ + +static inline void fl_push(struct hg_block *hb, void *p, unsigned int order) +{ + struct hg_free_blk *b = (struct hg_free_blk *)p; + + /* + * v3: the TOP-order list is kept in ascending address order, so the + * list head - what hg_buddy_alloc() serves whole pages from - is + * always the LOWEST free page. That is the whole prefer-low policy: + * new carves concentrate at the bottom, the top pages drain, and + * shrink (top-only by design) finds them whole-free. Lower orders + * stay LIFO - their blocks live inside pages already carved, where + * the cell-level concentration policy owns placement, and their + * lists are the long ones where an ordered walk would cost. + * + * The walk is bounded by the whole-free page count and runs on the + * buddy slow path only (a page reaches top order when it drains + * fully - GC and shrink territory, not the cell fast path). + */ + if (order == hb->buddy_top && hb->bfree[order]) { + struct hg_free_blk *cur = hb->bfree[order], *prev = NULL; + + while (cur && (void *)cur < p) { + prev = cur; + cur = cur->next; + } + b->prev = prev; + b->next = cur; + if (cur) + cur->prev = b; + if (prev) + prev->next = b; + else + hb->bfree[order] = b; + hb->nfree[order]++; + return; + } + + b->prev = NULL; + b->next = hb->bfree[order]; + if (b->next) + b->next->prev = b; + hb->bfree[order] = b; + hb->nfree[order]++; +} + +static inline void fl_unlink(struct hg_block *hb, void *p, unsigned int order) +{ + struct hg_free_blk *b = (struct hg_free_blk *)p; + + if (b->prev) + b->prev->next = b->next; + else + hb->bfree[order] = b->next; + if (b->next) + b->next->prev = b->prev; + hb->nfree[order]--; +} + +/* --- geometry --------------------------------------------------------- */ + +static inline struct hg_page *page_of(struct hg_block *hb, const void *p) +{ + return &hb->pages[hg_page_of(hb, p)]; +} + +/* --- init ------------------------------------------------------------- */ + +/* + * Publish one whole page as a single top-order free block. Only valid for a + * page nothing has been carved out of yet. + */ +static void page_publish_whole(struct hg_block *hb, struct hg_page *pg) +{ + unsigned int top = hb->buddy_top; + + memset(pg->leaforder, (int)top, (size_t)hg_leaves_per_page(hb)); + bit_set(pg->bitmap, node_id(top, top, 0)); + fl_push(hb, pg->base, top); + pg->free_leaves = (unsigned int)hg_leaves_per_page(hb); + hb->buddy_free_leaves += pg->free_leaves; +} + +/* + * Release the leaves [from, to) of a page that started out wholly reserved. + * + * Used for the one page that the block header and the buddy's own metadata + * partly occupy. Done leaf by leaf through the ordinary free path so the + * merges happen by the ordinary rules: freeing 24 consecutive leaves yields + * whatever mix of orders the alignment actually permits, which is fiddly to + * compute directly and trivial to get by construction. + * + * The alternative - refusing to use a partly-occupied page at all - would + * throw away up to a whole huge page. That is 0.8% of a 256 MB shm arena but + * 25% of an 8 MB pkg arena, which is not affordable. + */ +static void page_release_range(struct hg_block *hb, struct hg_page *pg, + unsigned long from, unsigned long to) +{ + unsigned long leaf; + + for (leaf = from; leaf < to; leaf++) { + /* hand it to the free path as a legitimately allocated leaf */ + pg->leaforder[leaf] = 0; + hg_buddy_free(hb, pg->base + (leaf << HG_LEAF_SHIFT), 0); + } +} + +int hg_buddy_init(struct hg_block *hb) +{ + unsigned long i, lpp, bmwords, meta, consumed_leaves; + unsigned int top; + char *meta_base, *cur; + struct hg_page *pg; + + if (hb->npages == 0) { + LM_INFO("%s: no whole pages, buddy reclaim inactive\n", hb->name); + return 0; + } + + top = hg_buddy_top_order(hb); + if (top > HG_MAX_ORDERS) { + LM_ERR("%s: %u buddy orders exceeds the %d the free-list array " + "holds\n", hb->name, top, HG_MAX_ORDERS); + return -1; + } + hb->buddy_top = top; + + lpp = hg_leaves_per_page(hb); + bmwords = (nodes_per_page(top) + sizeof(long) * 8 - 1) / + (sizeof(long) * 8); + + /* + * One contiguous metadata carve for all pages, from the FRONT of the + * arena via the ordinary bump allocator - so it inherits whatever tier + * the reservation achieved, is shared for shm and private for pkg with + * no decision to make, and costs no extra huge pages. A dedicated page + * would be 25% overhead on an 8 MB pkg arena, and 30 workers each + * wanting one would burn 60 MB to hold 45 KB. + * + * Sized for npages_cap, not npages: descriptors for pages the arena + * could GROW into are laid out now, from committed memory, so a later + * grow only publishes them - it never has to find room for metadata + * in an arena that is, by definition of why it is growing, full. The + * overhead is ~0.05% of each never-committed page, paid up front. + */ + meta = hb->npages_cap * (sizeof(struct hg_page) + lpp + + bmwords * sizeof(long)); + meta_base = hg_chunk_backing(hb, meta); + if (!meta_base) { + LM_ERR("%s: cannot carve %lu bytes of buddy metadata for %lu " + "pages (%lu committed)\n", hb->name, meta, + hb->npages_cap, hb->npages); + return -1; + } + memset(meta_base, 0, meta); + + hb->pages = (struct hg_page *)(void *)meta_base; + cur = meta_base + hb->npages_cap * sizeof(struct hg_page); + for (i = 0; i < hb->npages_cap; i++) { + pg = &hb->pages[i]; + pg->idx = (unsigned int)i; + pg->base = hb->pbase + (i << hb->hps_shift); + pg->leaforder = (unsigned char *)cur; + cur += lpp; + pg->bitmap = (unsigned long *)(void *)cur; + cur += bmwords * sizeof(long); + memset(pg->leaforder, HG_LEAF_NONE, lpp); + /* the committed pages carry init's achieved tier; pages beyond + * get theirs stamped by the grow that commits them */ + pg->tier = (unsigned char)hb->tier; + /* starts wholly reserved; pages < npages are published below, + * pages beyond wait for hg_buddy_grow() */ + } + + /* + * Everything below hoff is spoken for - the block header, then the + * metadata just carved. hoff is leaf aligned (hg_arena_init), so the + * boundary lands on a leaf and no partially-consumed leaf can be handed + * out. Round UP anyway: it costs at most one leaf and it means a future + * change to the bump allocator cannot silently start handing out memory + * that is already in use. + */ + consumed_leaves = 0; + { + unsigned long hoff = hb->hoff; + const char *cend = hb->hbase + ((hoff + HG_LEAF_SIZE - 1) & + ~(HG_LEAF_SIZE - 1)); + + if (cend > hb->pbase) + consumed_leaves = (unsigned long)(cend - hb->pbase) >> + HG_LEAF_SHIFT; + } + + for (i = 0; i < hb->npages; i++) { + unsigned long first = i * lpp, last = first + lpp; + + pg = &hb->pages[i]; + if (consumed_leaves >= last) + continue; /* wholly consumed */ + if (consumed_leaves <= first) { + page_publish_whole(hb, pg); /* wholly free */ + continue; + } + /* the single straddling page */ + page_release_range(hb, pg, consumed_leaves - first, lpp); + } + + /* + * Rebase the coalesce counter. Publishing the straddling page above + * goes through the ordinary free path on purpose - so the tree is built + * by the same rules it will be maintained by - but that means every one + * of those coalesces has just been counted, and they say nothing about + * fragmentation. Keep the total for the record and restart from zero, + * so buddy_splits and buddy_merges finally share a zero point: without + * this an idle 8 MB pkg arena reports 7 splits against 244 merges. + */ + hb->buddy_merges_init = hb->buddy_merges; + hb->buddy_merges = 0; + hb->buddy_splits = 0; /* init allocates nothing; make that explicit */ + + hb->buddy_ready = 1; + /* + * Reserve floor at 1/16 of the grid. A fraction rather than a constant + * because the arenas differ by three orders of magnitude - 8 MB pkg to + * 5 GB shm - and a fixed page count would be either meaningless on one + * or most of the other. + */ + hb->reserve_floor = (hb->npages * hg_leaves_per_page(hb)) / 16; + LM_DBG("%s buddy: %lu pages, orders 0..%u (%lu B..%lu B), %lu B metadata " + "(%lu B/page, %.3f%%), %lu of %lu leaves free after reserving %lu\n", + hb->name, hb->npages, top, HG_LEAF_SIZE, HG_LEAF_SIZE << top, + meta, meta / hb->npages, + 100.0 * (double)meta / (double)hb->hsize, + hb->buddy_free_leaves, hb->npages * lpp, consumed_leaves); + return 0; +} + +/* --- grow (v3) -------------------------------------------------------- */ + +/* + * Commit more of the reservation and publish the new whole pages. Called + * with hb->lock HELD, from the two places an allocation can die of buddy + * exhaustion (carve_chunk and the large tier), which also bounds how often + * it runs: once per granule of genuine demand, never on the fast path. + * + * The pre-fault inside hg_mem_commit() happens under the arena lock - a + * deliberate trade. Growth is rare (once per granule, ratcheting), the + * granule is sized to keep the stall in the low milliseconds, and the + * alternative - dropping the lock to fault, then re-taking it - opens a + * publish race for no benefit: every other worker in here is ALSO out of + * memory and would only queue on the same growth. + * + * Returns 0 if new pages were published (caller retries its allocation), + * -1 if the arena cannot grow (at cap, cap never set, or the commit was + * refused by the host). The caller's existing failure path then reports + * exhaustion exactly as a fixed arena would. + */ +/* + * A RESOURCE refusal - the host, not the admin, said no. Counts, and runs + * the two-step latch described on grow_blocked's declaration: arm on the + * first refusal, latch only if the arena is refused again after a full GC + * pass ran - a spike that reclaim absorbs never alerts. The latch WARNs + * once and hands the event raise to the sweep timer via grow_event_due; + * nothing is raised from here, hb->lock is held. + */ +static int grow_resource_refused(struct hg_block *hb) +{ + hb->grow_refused++; + + if (hb->grow_blocked) + return -1; + + if (!hb->grow_blocked_mark) { + /* gc_passes + 1 doubles as the "armed" flag: it can never be + * 0, and it is exactly the count a pass must push gc_passes + * PAST for the refusal to have survived one */ + hb->grow_blocked_mark = hb->gc_passes + 1; + hb->grow_blocked_refuse0 = hb->grow_refused; + } else if (hb->gc_passes >= hb->grow_blocked_mark) { + hb->grow_blocked = 1; + hb->grow_event_due = 1; + LM_WARN("%s: GROW-BLOCKED latched - the arena cannot grow and " + "a GC pass did not change that (%lu refusals so far). " + "Alert on hg_shm_grow_blocked; details precede this " + "line.\n", hb->name, hb->grow_refused); + } + return -1; +} + +/* the blocked state ends two ways; both say so if there is anything to + * end, and both re-arm the once-per-episode messages */ +void hg_grow_unblock(struct hg_block *hb, const char *how) +{ + if (hb->grow_blocked) + LM_NOTICE("%s: GROW-BLOCKED cleared - %s\n", hb->name, how); + hb->grow_blocked = 0; + hb->grow_blocked_mark = 0; + hb->grow_blocked_refuse0 = 0; + hb->grow_event_due = 0; + hb->grow_refuse_said = 0; +} + +/* + * The sweep timer's half of the latch - see grow_blocked_refuse0's + * declaration for why the GC route alone cannot be trusted. Called once + * per sweep interval with hb->lock HELD; latches if an armed episode is + * still accumulating refusals a full interval later. + */ +void hg_grow_blocked_tick(struct hg_block *hb) +{ + if (hb->grow_blocked || !hb->grow_blocked_mark) + return; + if (hb->grow_refused > hb->grow_blocked_refuse0) { + hb->grow_blocked = 1; + hb->grow_event_due = 1; + LM_WARN("%s: GROW-BLOCKED latched - the arena cannot grow and " + "a full sweep interval did not change that (%lu refusals " + "so far). Alert on hg_shm_grow_blocked; details precede " + "this line.\n", hb->name, hb->grow_refused); + } else { + /* armed but quiet for a whole interval: the spike passed */ + hb->grow_blocked_mark = 0; + hb->grow_blocked_refuse0 = 0; + } +} + +int hg_buddy_grow(struct hg_block *hb, unsigned long need) +{ + unsigned long delta, room, old_pages, i, limit; + int tier; + + if (!hb->buddy_ready) + return -1; + + /* advise-only mode: report what growth WOULD have done, act never - + * the arena behaves exactly like a fixed v2 one, with evidence */ + if (hg_autoscale_dry_run) { + hb->grow_refused++; + if (!hb->pol_dry_said) { + hb->pol_dry_said = 1; + LM_WARN("%s: DRY RUN - would grow for a %lu byte request " + "(committed %lu MB); counting further suppressed " + "grows in hg_shm_grow_refused\n", + hb->name, need, hb->hsize >> 20); + } + return -1; + } + + /* the profile's scale-up target is the admin ceiling WITHIN the + * -m INIT:CAP reservation; without a profile the reservation is the + * ceiling */ + limit = (hb->pol.active && hb->pol.up_bytes) ? hb->pol.up_bytes + : hb->hcap; + room = limit > hb->hsize ? limit - hb->hsize : 0; + if (room == 0) { + hb->grow_refused++; + /* an admin-set ceiling doing its job is not an alarm; growth + * being impossible because no cap was ever set is not even + * noteworthy - v2 arenas live their whole lives there. The two + * are told apart by history, not arithmetic: a growable arena + * can only reach room==0 by having grown (hsize starts below + * hcap and moves only in grows), so grows>0 here means "the + * headroom existed and is spent", while grows==0 means the + * arena never had any. Said once per episode - grow_refused + * carries the magnitude. */ + if (hb->grows && !hb->grow_refuse_said) { + hb->grow_refuse_said = 1; + LM_NOTICE("%s: at the %lu MB growth ceiling (%s), " + "a %lu byte request must fail - counting further " + "refusals in hg_shm_grow_refused\n", + hb->name, limit >> 20, + limit == hb->hcap ? "the -m/-M reservation" + : "the profile scale-up target", + need); + } + return -1; + } + + delta = hb->grow_granule; + if (need > delta) + delta = (need + hb->grow_granule - 1) / + hb->grow_granule * hb->grow_granule; + if (delta > room) + delta = room; + + /* the host-RAM limb of the ceiling, before any work is done */ + if (hg_grow_ram_refused(hb, delta)) + return grow_resource_refused(hb); + + tier = hg_mem_commit(hb, hb->hsize, delta); + if (tier < 0) { + /* the commit rolled itself back; nothing was published */ + return grow_resource_refused(hb); + } + + hb->tier_bytes[tier] += delta; + hb->hsize += delta; + /* hb->size is the figure every "total/free" surface reports (shmem + * statistics, hg_info, hg_advise's configured_mb) and free_to_carve + * is literally size - real_used: leave it behind and that subtraction + * underflows once carving passes the original size. The per-thread + * cache budget and chunk_max stay on their init-time derivation - + * conservative, and re-deriving them per grow would change cell-cache + * behaviour mid-flight for a marginal win. */ + hb->size += delta; + old_pages = hb->npages; + hb->npages = (unsigned long)(hb->hbase + hb->hsize - hb->pbase) + >> hb->hps_shift; + + for (i = old_pages; i < hb->npages; i++) { + hb->pages[i].tier = (unsigned char)tier; + page_publish_whole(hb, &hb->pages[i]); + } + + /* keep the floor at 1/16 of the grid it now guards */ + hb->reserve_floor = (hb->npages * hg_leaves_per_page(hb)) / 16; + + hb->grows++; + hb->grow_bytes += delta; + hb->shrink_quiet = 0; /* fresh demand voids any quiet window */ + hb->pol_cooldown = hb->pol.active ? hb->pol.cooldown : 0; + hb->pol_dry_said = 0; + hg_grow_unblock(hb, "the arena grew, the resource came back"); + + LM_NOTICE("%s arena grew by %lu MB to %lu MB (%lu new pages on %s; " + "%lu MB headroom left)\n", hb->name, delta >> 20, + hb->hsize >> 20, hb->npages - old_pages, + hg_mem_tier_str((enum hg_mem_tier)tier), + (hb->hcap - hb->hsize) >> 20); + return 0; +} + +/* --- shrink (v3) ------------------------------------------------------ */ + +/* defined with the run machinery below; shrink shares its eligibility test */ +static inline int page_is_whole_free(const struct hg_block *hb, + const struct hg_page *pg); + +/* + * Release up to one granule of whole-free pages from the TOP of the + * committed range. Top-only is what keeps every address invariant intact: + * hg_owns() stays one contiguous test, the registry entry stays valid, and + * a page below the new top is untouched. Whole-free is what makes it SAFE + * with no cross-process coordination: eager merging guarantees a + * whole-free page is one top-order block on the free list, and a cell + * parked in some thread's private cache has NOT decremented its block's + * live count - so its page is not whole-free and can never be picked here. + * + * Never below hsize_min: the admin asked for -m/-M; only growth is + * elastic. The release syscall runs BEFORE any bookkeeping, while + * hb->lock (held by the caller) keeps every allocator out of the pages + * being punched - if the kernel refuses, nothing has changed. + */ +static void hg_buddy_shrink(struct hg_block *hb) +{ + unsigned long lpp = hg_leaves_per_page(hb); + unsigned long limit = hb->grow_granule >> hb->hps_shift; + unsigned long n = 0, i, off, len; + unsigned int top = hb->buddy_top; + + while (n < limit && + hb->hsize - ((n + 1UL) << hb->hps_shift) >= hb->hsize_min && + page_is_whole_free(hb, &hb->pages[hb->npages - 1 - n])) + n++; + if (!n) + return; + + len = n << hb->hps_shift; + off = (unsigned long)(hb->pages[hb->npages - n].base - hb->hbase); + if (off + len != hb->hsize) { + /* a platform where the grid does not end exactly at the + * committed end (unaligned non-Linux base). Shrinking a + * mid-range is correct for the punch but wrong for the + * hsize arithmetic - decline rather than approximate. */ + return; + } + + if (hg_mem_release(hb, off, len) != 0) + return; + + for (i = hb->npages - n; i < hb->npages; i++) { + struct hg_page *pg = &hb->pages[i]; + + fl_unlink(hb, pg->base, top); + bit_clear(pg->bitmap, node_id(top, top, 0)); + memset(pg->leaforder, HG_LEAF_NONE, (size_t)lpp); + pg->free_leaves = 0; + pg->run_len = 0; + hb->buddy_free_leaves -= lpp; + if (hb->tier_bytes[pg->tier] >= hb->hps) + hb->tier_bytes[pg->tier] -= hb->hps; + } + hb->npages -= n; + hb->hsize -= len; + hb->size -= len; + hb->shrinks++; + hb->shrink_bytes += len; + hb->reserve_floor = (hb->npages * lpp) / 16; + + LM_NOTICE("%s arena shrank by %lu MB to %lu MB (%lu pages released " + "to the %s; %lu MB of growth still held)\n", hb->name, + len >> 20, hb->hsize >> 20, n, + hb->tier == HG_MEM_HUGETLB ? "hugetlb pool" : "host", + (hb->hsize - hb->hsize_min) >> 20); +} + +/* the no-policy default: consecutive quiet sweep ticks per released + * granule (two minutes at the 30 s sweep) - deliberately down-slow. A + * profile replaces this with its own "for N cycles". */ +#define HG_SHRINK_QUIET_TICKS 4 + +/* + * The down-slow policy gate, one call per sweep interval, hb->lock held. + * Counts a tick as "quiet" only while ALL of it holds: elastic bytes + * exist, nothing is starved (not below the floor, not grow-blocked), the + * top page is already whole-free, and free space would stay generously + * clear of the floor's recovery threshold even after giving a granule + * back - so a shrink can never be the thing that re-triggers pressure. + * Any failed condition resets the window; so does any grow. + * + * These thresholds are the hardcoded seed of the scale-down half of the + * auto_scaling_profile surface; the profile replaces the constants, not + * the shape. + */ +void hg_shrink_tick(struct hg_block *hb) +{ + unsigned long granule_leaves; + unsigned int need_ticks; + + if (!hb->buddy_ready || hb->shrink_unsupported) + return; + if (hb->hsize <= hb->hsize_min) { + hb->shrink_quiet = 0; + return; + } + /* post-grow cool-off: the profile grammar's 10x-cycles hold, so an + * arena that just grew cannot immediately give the growth back */ + if (hb->pol_cooldown) { + hb->pol_cooldown--; + hb->shrink_quiet = 0; + return; + } + /* the hard SAFETY conditions hold with or without a policy: never + * shrink an arena that is starved, latched, or whose top page is + * still in use */ + if (hb->below_floor || hb->grow_blocked || + !page_is_whole_free(hb, &hb->pages[hb->npages - 1])) { + hb->shrink_quiet = 0; + return; + } + if (hb->pol.active) { + /* the profile's own quiet test: usage at or below its + * down-threshold, plus the giving-a-granule-back-stays-safe + * floor guard */ + if (hb->real_used * 100 > (unsigned long)hb->pol.down_pct * + hb->hsize || + hb->buddy_free_leaves < + (hb->grow_granule >> HG_LEAF_SHIFT) + + hb->reserve_floor * 2) { + hb->shrink_quiet = 0; + return; + } + need_ticks = hb->pol.down_cycles ? hb->pol.down_cycles : 1; + } else { + granule_leaves = hb->grow_granule >> HG_LEAF_SHIFT; + if (hb->buddy_free_leaves < + granule_leaves + hb->reserve_floor * 4) { + hb->shrink_quiet = 0; + return; + } + need_ticks = HG_SHRINK_QUIET_TICKS; + } + if (++hb->shrink_quiet < need_ticks) + return; + hb->shrink_quiet = 0; + if (hg_autoscale_dry_run) { + if (!hb->pol_dry_said) { + hb->pol_dry_said = 1; + LM_NOTICE("%s: DRY RUN - would shrink (committed %lu MB, " + "usage %lu%%)\n", hb->name, hb->hsize >> 20, + hb->real_used * 100 / hb->hsize); + } + return; + } + hg_buddy_shrink(hb); +} + +/* + * The proactive half of the profile: grow BEFORE exhaustion when usage + * has crossed the up-threshold often enough. Same call sites and lock + * contract as hg_shrink_tick(); a profile-less arena never enters (its + * growth remains exhaustion-triggered, the step-1 emergency path, which + * also stays armed WITH a profile - a burst between ticks must not fail + * allocations while the timer catches up). + */ +void hg_grow_tick(struct hg_block *hb) +{ + int hit; + + if (!hb->buddy_ready || !hb->pol.active) + return; + if (hb->hsize >= hb->pol.up_bytes) + return; /* at the profile ceiling */ + + hb->pol_up_ticks++; + if (hb->real_used * 100 >= (unsigned long)hb->pol.up_pct * hb->hsize) + hb->pol_up_hits++; + + if (hb->pol_up_ticks < + (hb->pol.up_window ? hb->pol.up_window : 1)) + return; + hit = hb->pol_up_hits >= (hb->pol.up_need ? hb->pol.up_need : 1); + hb->pol_up_ticks = 0; + hb->pol_up_hits = 0; + if (!hit) + return; + + if (hg_autoscale_dry_run) { + if (!hb->pol_dry_said) { + hb->pol_dry_said = 1; + LM_NOTICE("%s: DRY RUN - would grow (committed %lu MB, " + "usage %lu%%, profile ceiling %lu MB)\n", + hb->name, hb->hsize >> 20, + hb->real_used * 100 / hb->hsize, + hb->pol.up_bytes >> 20); + } + return; + } + hg_buddy_grow(hb, hb->grow_granule); +} + +/* --- allocate --------------------------------------------------------- */ + +void *hg_buddy_alloc(struct hg_block *hb, unsigned int order) +{ + unsigned int o, top = hb->buddy_top; + struct hg_page *pg; + unsigned long leaf; + char *blk; + + if (!hb->buddy_ready || order > top) + return NULL; + + /* + * Smallest free block that fits, so large free blocks are preserved by + * construction (design, "allocation policy"). Scanning UP from the + * requested order is exactly that: the first non-empty list is the + * smallest one that can serve it. + */ + for (o = order; o <= top; o++) + if (hb->bfree[o]) + break; + if (o > top) + return NULL; + + blk = (char *)hb->bfree[o]; + fl_unlink(hb, blk, o); + pg = page_of(hb, blk); + leaf = hg_leaf_of(hb, blk); + bit_clear(pg->bitmap, node_id(top, o, leaf)); + + /* split down, publishing the upper half at each step. The lower half + * stays in hand, so the returned address never moves. */ + while (o > order) { + unsigned long bleaf; + char *buddy; + + o--; + bleaf = leaf + (1UL << o); + buddy = pg->base + (bleaf << HG_LEAF_SHIFT); + memset(pg->leaforder + bleaf, (int)o, (size_t)1UL << o); + bit_set(pg->bitmap, node_id(top, o, bleaf)); + fl_push(hb, buddy, o); + hb->buddy_splits++; + } + + memset(pg->leaforder + leaf, (int)order, (size_t)1UL << order); + pg->free_leaves -= 1U << order; + hb->buddy_free_leaves -= 1UL << order; + hg_extent_note(hb, blk, (unsigned long)HG_LEAF_SIZE << order); + hg_reserve_floor_check(hb); + return blk; +} + +/* --- free ------------------------------------------------------------- */ + +void hg_buddy_free(struct hg_block *hb, void *p, unsigned int order) +{ + unsigned int o = order, top = hb->buddy_top; + struct hg_page *pg; + unsigned long leaf; + char *blk = p; + + if (!hg_in_pages(hb, p)) { + hg_corrupt(hb, HG_C_BUDDY_BAD_FREE); + LM_CRIT("%s: buddy free of %p, which is outside the page grid - " + "ignoring\n", hb->name, p); + return; + } + pg = page_of(hb, p); + leaf = hg_leaf_of(hb, p); + + if (((unsigned long)p & ((HG_LEAF_SIZE << order) - 1)) != + ((unsigned long)pg->base & ((HG_LEAF_SIZE << order) - 1))) { + hg_corrupt(hb, HG_C_BUDDY_BAD_FREE); + LM_CRIT("%s: buddy free of %p at order %u, which is not aligned to " + "its own size - ignoring\n", hb->name, p, order); + return; + } + if (leaf & ((1UL << order) - 1)) { + hg_corrupt(hb, HG_C_BUDDY_BAD_FREE); + LM_CRIT("%s: buddy free of %p as order %u, but leaf %lu does not " + "start a block of that order - ignoring\n", + hb->name, p, order, leaf); + return; + } + if (pg->leaforder[leaf] != order) { + hg_corrupt(hb, HG_C_BUDDY_BAD_FREE); + LM_CRIT("%s: buddy free of %p as order %u, but leaf %lu records " + "order %u - ignoring\n", hb->name, p, order, leaf, + pg->leaforder[leaf]); + return; + } + /* + * Double free. The leaforder check above does NOT catch it: a block that + * failed to merge still records its own order, so freeing it twice would + * look entirely legitimate and push it onto the free list a second time, + * after which two callers get the same address. The bitmap is the + * authority on "already free and entire", which is precisely this. + */ + if (bit_test(pg->bitmap, node_id(top, order, leaf))) { + hg_corrupt(hb, HG_C_DOUBLE_FREE); + LM_CRIT("%s: double buddy free of %p at order %u - ignoring\n", + hb->name, p, order); + return; + } + + pg->free_leaves += 1U << order; + hb->buddy_free_leaves += 1UL << order; + + /* + * Merge upwards while the buddy is free and entire. The buddy's address + * is this block's with one bit flipped, which is what keeps each step + * O(1); the loop runs at most `top` times. + * + * Note the merge stops at the page. The top order IS the page, so there + * is no cross-page merging to implement and a wholly free top block is + * exactly one huge page - which is the unit the reclaim in task #57 will + * hand back. + */ + while (o < top) { + unsigned long bleaf = leaf ^ (1UL << o); + char *buddy = pg->base + (bleaf << HG_LEAF_SHIFT); + + if (!bit_test(pg->bitmap, node_id(top, o, bleaf))) + break; /* allocated, or split */ + if (pg->leaforder[bleaf] != o) + break; /* free but not at this order */ + + fl_unlink(hb, buddy, o); + bit_clear(pg->bitmap, node_id(top, o, bleaf)); + hb->buddy_merges++; + pg->leaforder[bleaf] = HG_LEAF_NONE; + + if (bleaf < leaf) { /* we are the upper half - move down */ + leaf = bleaf; + blk = buddy; + } + o++; + } + + memset(pg->leaforder + leaf, (int)o, (size_t)1UL << o); + bit_set(pg->bitmap, node_id(top, o, leaf)); + fl_push(hb, blk, o); + hg_reserve_floor_check(hb); +} + +/* --- multi-page runs -------------------------------------------------- */ + +/* is this whole page free and unsplit, i.e. available to a run? */ +static inline int page_is_whole_free(const struct hg_block *hb, + const struct hg_page *pg) +{ + unsigned int top = hb->buddy_top; + + return pg->run_len == 0 && bit_test(pg->bitmap, node_id(top, top, 0)) && + pg->leaforder[0] == top; +} + +void *hg_buddy_alloc_run(struct hg_block *hb, unsigned long npages) +{ + unsigned int top = hb->buddy_top; + unsigned long i, start, run = 0; + + if (!hb->buddy_ready || npages == 0) + return NULL; + if (npages == 1) + return hg_buddy_alloc(hb, top); + + for (i = 0, start = 0; i < hb->npages; i++) { + if (!page_is_whole_free(hb, &hb->pages[i])) { + run = 0; + start = i + 1; + continue; + } + if (++run == npages) + break; + } + if (run < npages) + return NULL; + + for (i = start; i < start + npages; i++) { + struct hg_page *pg = &hb->pages[i]; + + fl_unlink(hb, pg->base, top); + bit_clear(pg->bitmap, node_id(top, top, 0)); + memset(pg->leaforder, HG_LEAF_NONE, + (size_t)hg_leaves_per_page(hb)); + pg->free_leaves = 0; + pg->run_len = (i == start) ? (unsigned int)npages : HG_RUN_MEMBER; + hb->buddy_free_leaves -= hg_leaves_per_page(hb); + } + hg_extent_note(hb, hb->pages[start].base, npages << hb->hps_shift); + hg_reserve_floor_check(hb); + LM_DBG("%s: run of %lu pages at page %lu\n", hb->name, npages, start); + return hb->pages[start].base; +} + +unsigned long hg_buddy_run_len(const struct hg_block *hb, const void *p) +{ + const struct hg_page *pg; + + if (!hb->buddy_ready || !hg_in_pages(hb, p)) + return 0; + pg = &hb->pages[hg_page_of(hb, p)]; + if (pg->base != p || pg->run_len == 0 || pg->run_len == HG_RUN_MEMBER) + return 0; + return pg->run_len; +} + +void hg_buddy_free_run(struct hg_block *hb, void *p) +{ + unsigned long n, i, start; + struct hg_page *pg; + + n = hg_buddy_run_len(hb, p); + if (n == 0) { + hg_corrupt(hb, HG_C_BUDDY_BAD_FREE); + LM_CRIT("%s: run free of %p, which heads no run - ignoring\n", + hb->name, p); + return; + } + start = hg_page_of(hb, p); + for (i = start; i < start + n; i++) { + pg = &hb->pages[i]; + pg->run_len = 0; + page_publish_whole(hb, pg); + } + hg_reserve_floor_check(hb); +} + +int hg_buddy_order_of(const struct hg_block *hb, const void *p) +{ + const struct hg_page *pg; + unsigned long leaf; + + if (!hb->buddy_ready || !hg_in_pages(hb, p)) + return -1; + pg = &hb->pages[hg_page_of(hb, p)]; + leaf = hg_leaf_of(hb, p); + if (pg->leaforder[leaf] == HG_LEAF_NONE) + return -1; + return pg->leaforder[leaf]; +} + +#endif /* HG_MALLOC */ diff --git a/mem/hg_buddy.h b/mem/hg_buddy.h new file mode 100644 index 00000000000..2bf02557ea9 --- /dev/null +++ b/mem/hg_buddy.h @@ -0,0 +1,233 @@ +/* + * buddy allocator over the HG_MALLOC huge-page grid + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * One buddy tree per huge page - see mem/README.hg_arena_v2, "Structure". + * + * Order 0 is the leaf (HG_LEAF_SIZE, 8 KB); the top order is the whole page, + * so the hierarchy terminates at the page and there is no cross-page merging + * to implement. A wholly free top block IS one huge page. + * + * Splitting is pure bookkeeping: the arena is already mapped and pre-faulted + * at init, so turning a 16 KB block into two 8 KB blocks changes records + * only. Merging is the reverse and is the only legal defragmentation here, + * because it moves free space rather than live objects (constraint 1 of the + * design: callers hold raw pointers, so nothing live can ever be relocated). + * + * Everything in here runs under hb->lock, on the slow path that already takes + * it. Nothing on the cell fast path calls into this file. + */ + +#ifndef HG_BUDDY_H +#define HG_BUDDY_H + +#include "hg_malloc.h" + +/* HG_MAX_ORDERS lives in hg_malloc.h - struct hg_block needs it for the + * free-list array, and this header includes that one, not the other way. */ + +/* A free block stores its own list linkage in its first bytes. Legitimate + * because the block is free - nothing else is using those bytes - and it is + * what keeps the free lists free of external metadata. The smallest block is + * HG_LEAF_SIZE, vastly larger than this struct. + * + * There was a third field here, a "buddyfr" magic stamped by fl_push and + * cleared by fl_unlink, meant to catch a double free or a wild pointer. It was + * never read - the tree contained the two writes and nothing else - and it is + * gone rather than completed, because both things it promised are already + * caught, and caught better: + * + * double free the BITMAP, hg_buddy_free() -> bit_test(pg->bitmap, ...). + * See the comment there for why it, and not the recorded + * order, is the authority on "already free and entire". + * wild pointer hg_buddy_free()'s alignment, order and page-bounds refusals, + * five more hg_corrupt() sites in the same function. + * + * A magic living INSIDE the freed block would also have been the weaker of the + * two: a caller scribbling on memory it already freed can forge it, while the + * bitmap sits in the metadata region carved from the front of the arena and + * cannot be reached that way. */ +struct hg_free_blk { + struct hg_free_blk *next; + struct hg_free_blk *prev; +}; + +/* + * Per-page descriptor. Lives in the metadata region carved from the front of + * the arena, NOT inside the page it describes - a page must be able to become + * wholly free, and it cannot if its own bookkeeping sits in it. + */ +struct hg_page { + char *base; /* first byte of this page */ + unsigned char *leaforder; /* per leaf: order of the block starting here */ + unsigned long *bitmap; /* per tree node: 1 = free, whole, not split */ + + /* Fullness lists. Unused until the allocation policy lands (task #58, + * "page preference"); kept here because the descriptor is sized to the + * design's 64 byte budget and these two fit inside it. */ + struct hg_page *next; + struct hg_page *prev; + + unsigned int idx; /* page index within the arena */ + unsigned int free_leaves; /* leaves not currently allocated */ + /* >0 on the FIRST page of a multi-page run, giving its length; the + * other pages of the run carry HG_RUN_MEMBER. See hg_buddy_alloc_run(). */ + unsigned int run_len; + /* v3: the ACHIEVED backing tier of the commit that brought this page + * in (enum hg_mem_tier). Growth deltas negotiate their own backing, + * and shrink is top-only over pages from arbitrary deltas - without + * this byte the tier_bytes histogram could not be decremented + * truthfully. Fits the existing padding; the descriptor stays 56. */ + unsigned char tier; +}; + +/* run_len marker for a page that belongs to a run but does not head it */ +#define HG_RUN_MEMBER 0xffffffffu +/* Deliberately NOT carrying a per-order free count per page: at HG_MAX_ORDERS + * that array alone is 100 bytes and would take the descriptor from 48 to 148, + * over double the design's 64 byte budget and, on a 5 GB arena, from 160 KB to + * 370 KB of descriptors. Per-order counts are global, on hg_block. */ + +/* leaforder[] value for a leaf that does not START a block (it is in the + * middle of a larger one, or its block is allocated and recorded elsewhere) */ +#define HG_LEAF_NONE 0xff + +/* + * Set up the page grid's buddy state. Carves the metadata for every page from + * the front of the arena, marks the region already consumed by the block + * header and that metadata as allocated, and publishes the rest as free + * blocks. Call once, from hg_arena_init(), after pages_init(). + * + * Returns 0 on success, -1 if the metadata cannot be carved. + */ +int hg_buddy_init(struct hg_block *hb); + +/* Allocate one block of exactly (HG_LEAF_SIZE << order) bytes, naturally + * aligned. hb->lock must be held. Returns NULL when no page can serve it. */ +void *hg_buddy_alloc(struct hg_block *hb, unsigned int order); + +/* v3: commit up to max(granule, @need) more of the reservation and publish + * the new whole pages as free. hb->lock must be held. 0 = grew (retry the + * allocation), -1 = cannot (at cap / no cap / host refused the commit). */ +int hg_buddy_grow(struct hg_block *hb, unsigned long need); + +/* v3: end a grow-blocked episode (say so if one was latched) and re-arm + * the once-per-episode reporting. hb->lock must be held. Called by the + * grow that succeeds and by the reserve floor's recovery branch - the + * "demand fell below the lower mark" clear. @how finishes the sentence + * "GROW-BLOCKED cleared - ". */ +void hg_grow_unblock(struct hg_block *hb, const char *how); + +/* v3: the sweep timer's promoter for an armed grow-blocked episode - the + * GC-pass route cannot latch on an arena where nothing is reclaimable. + * hb->lock must be held. Also disarms an episode that went quiet. */ +void hg_grow_blocked_tick(struct hg_block *hb); + +/* v3: the down-slow shrink gate, once per sweep interval per arena, + * hb->lock held. After HG_SHRINK_QUIET_TICKS consecutive quiet ticks it + * releases up to one granule of whole-free TOP pages back to the host + * (or, tier 1, the hugetlb pool) - never below the initial -m/-M size. + * Called for shm from the sweep timer and for pkg from each process's + * own flush path: a pkg arena is private, only its owner can shrink it. */ +void hg_shrink_tick(struct hg_block *hb); + +/* v3 step 4: the profile's proactive grow gate - same cadence, call sites + * and lock contract as hg_shrink_tick(). No-op without a policy. */ +void hg_grow_tick(struct hg_block *hb); + +/* Return a block previously handed out by hg_buddy_alloc(), merging it with + * its buddy as far up as it will go. hb->lock must be held. */ +void hg_buddy_free(struct hg_block *hb, void *p, unsigned int order); + +/* Order of the block containing @p, or -1 if @p is not in buddy space. */ +int hg_buddy_order_of(const struct hg_block *hb, const void *p); + +/* + * The block CONTAINING @p, from any address inside it - two shifts, a table + * byte and a mask, no search. This is the lookup the design is built around: + * a cell being freed sits somewhere in the middle of its block, and reaching + * the block is the only way its live count can ever be decremented, which is + * the only way a block can ever be recognised as empty and reclaimed. + * + * Returns NULL if @p is outside the page grid or in a multi-page run. + */ +static inline void *hg_buddy_block_of(const struct hg_block *hb, const void *p) +{ + const struct hg_page *pg; + unsigned long leaf; + unsigned char o; + + if (!hb->buddy_ready || !hg_in_pages(hb, p)) + return NULL; + pg = &hb->pages[hg_page_of(hb, p)]; + leaf = hg_leaf_of(hb, p); + o = pg->leaforder[leaf]; + if (o == HG_LEAF_NONE) + return NULL; + return pg->base + ((leaf & ~((1UL << o) - 1)) << HG_LEAF_SHIFT); +} + +/* + * A run of @npages CONTIGUOUS whole pages, for the one thing the tree cannot + * express: an allocation larger than a huge page. The large tier needs it - + * a dialog or usrloc hash table is routinely several MB in one piece - and + * before the buddy those went to the bump allocator, which had no upper bound. + * Losing that would be a regression, not a simplification. + * + * Deliberately a linear scan for a free run, not an index: it is reached only + * when the large tier grows past a page, the scan is over page descriptors + * (2560 of them on a 5 GB arena) and it happens orders of magnitude less often + * than a cell allocation. Paying for an index here would be paying for + * nothing. + * + * hb->lock must be held. Returns the run's first byte, or NULL. + */ +void *hg_buddy_alloc_run(struct hg_block *hb, unsigned long npages); + +/* Release a run obtained from hg_buddy_alloc_run(). hb->lock must be held. */ +void hg_buddy_free_run(struct hg_block *hb, void *p); + +/* Length in pages of the run starting at @p, or 0 if @p heads no run. */ +unsigned long hg_buddy_run_len(const struct hg_block *hb, const void *p); + +/* Highest order this arena can serve, i.e. the whole-page order. */ +static inline unsigned int hg_buddy_top_order(const struct hg_block *hb) +{ + return hb->hps_shift - HG_LEAF_SHIFT; +} + +/* Smallest order whose block is at least @bytes. Returns -1 if @bytes exceeds + * a whole page, which the caller must route to the large tier instead. */ +static inline int hg_buddy_order_for(const struct hg_block *hb, + unsigned long bytes) +{ + unsigned int o = 0; + + while ((HG_LEAF_SIZE << o) < bytes) { + if (o >= hg_buddy_top_order(hb)) + return -1; + o++; + } + return (int)o; +} + +#endif /* HG_BUDDY_H */ diff --git a/mem/hg_large.c b/mem/hg_large.c new file mode 100644 index 00000000000..d30ff7d78f7 --- /dev/null +++ b/mem/hg_large.c @@ -0,0 +1,420 @@ +/* + * hugepage-backed slab allocator - large-object tier + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef HG_MALLOC + +#include "hg_malloc.h" +#include "hg_large.h" +#include "hg_arena.h" +#include "hg_buddy.h" +#include "../dprint.h" + +#define HG_LARGE_MIN_FRAG HG_ROUNDTO +#define HG_LARGE_DEFAULT_CHUNK (1UL << 20) /* 1 MB - amortizes future churn */ + +struct hg_large_chunk { + struct hg_large_chunk *next; + unsigned long size; + /* the buddy block/run this chunk occupies, INCLUDING this header and + * any rounding slack - what the grid actually lost, as opposed to + * ->size, which is only the part available to fragments */ + unsigned long backing; + /* how it was taken, so it can be given back the same way: a buddy + * block of this order, or -1 for a run of whole pages */ + int ord; + struct hg_lfrag *first_frag; + struct hg_lfrag *last_frag; /* sentinel: size=0, prev always NULL, + * naturally stops forward coalescing at + * the chunk boundary without a separate + * bounds check */ +}; + +/* size AFTER the hg_lfrag header (tag region + payload for a live frag, + * or free-list linkage for a free one) - same convention as f_malloc's + * fm_frag->size, so FRAG_NEXT-style arithmetic matches */ +#define HG_LFRAG_NEXT(f) \ + ((struct hg_lfrag *)(void *)((char *)(f) + HG_LFRAG_HDR + (f)->size)) + +static inline void lfrag_insert_free(struct hg_block *hb, struct hg_lfrag *frag) +{ + frag->prev = &hb->large_free; + frag->nxt_free = hb->large_free; + if (hb->large_free) + hb->large_free->prev = &frag->nxt_free; + hb->large_free = frag; +} + +static inline void lfrag_remove_free(struct hg_lfrag *frag) +{ + *(frag->prev) = frag->nxt_free; + if (frag->nxt_free) + frag->nxt_free->prev = frag->prev; + frag->prev = NULL; +} + +#ifdef DBG_MALLOC +void *hg_large_alloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line) +#else +void *hg_large_alloc(struct hg_block *hb, unsigned long size) +#endif +{ + unsigned long need, chunk_size, rest, backing = 0; + int chunk_ord = -1; + struct hg_lfrag *f, *n; + struct hg_large_chunk *ch; + /* base MUST start NULL: the carve below is a grow-and-retry loop whose + * condition reads it before the first attempt assigns it. gcc 9 does + * not flag the uninitialized read this replaced (caught live: the + * garbage "pointer" was &evi_time_str+12, and the lock_init loop it + * was handed to overwrote log_level) */ + char *base = NULL, *tag; + + /* round to HG_PAYLOAD_ALIGN, not HG_ROUNDTO: a frag's size is what + * places the NEXT frag (HG_LFRAG_NEXT), so rounding to 4 on 32-bit + * ARM would walk every subsequent frag - and its payload - off the + * 8-byte boundary the first one started on */ + need = (HG_CELL_HDR + size + HG_PAYLOAD_ALIGN - 1) + & ~(unsigned long)(HG_PAYLOAD_ALIGN - 1); + + lock_get(&hb->lock); + + /* linear first-fit across ONE shared, unsorted free list (large + * allocations are inherently rare and already slow-path - see the + * hg_large.h file header for why this doesn't need f_malloc's + * size-hashed buckets) */ + f = NULL; + for (n = hb->large_free; n; n = n->nxt_free) { + if (n->size >= need) { + f = n; + break; + } + } + + if (!f) { + /* grow: carve a new chunk from the SAME hoff arena the + * small-object tier uses, sized for this request (plus a + * default floor so small large-object churn doesn't force a + * fresh chunk carve every time) */ + chunk_size = HG_LFRAG_HDR * 2 + need; + /* the 1M amortisation floor is only affordable on a big arena; + * on a small one it would eat the whole thing (same reasoning + * as chunk_size_for() in hg_arena.c) */ + { + unsigned long floor = HG_LARGE_DEFAULT_CHUNK; + if (floor > hb->chunk_max) + floor = hb->chunk_max; + if (chunk_size < floor) + chunk_size = floor; + } + chunk_size = (chunk_size + 63) & ~63UL; + + /* + * Large chunks come from the buddy now, not the bump allocator. + * Two shapes, because the tree tops out at one huge page: + * + * up to a page - one buddy block, rounded up to its order. The + * slack is not lost, it becomes free-list space + * inside the chunk. + * over a page - a run of contiguous whole pages. Rare, but it + * MUST work: a dialog or usrloc hash table is one + * allocation of several MB, and the bump allocator + * this replaces had no upper bound at all. + */ + { + unsigned long total = chunk_size + sizeof(struct hg_large_chunk); + int ord = hg_buddy_order_for(hb, total); + int attempt; + + /* two passes: a miss on the first is a growth trigger + * (v3), and the retry after a successful grow is served + * from whole fresh pages by construction. A second miss + * means the grow was refused - fall through to the + * ordinary exhaustion error. */ + for (attempt = 0; attempt < 2 && !base; attempt++) { + if (attempt && hg_buddy_grow(hb, total) != 0) + break; + if (ord >= 0) { + base = hg_buddy_alloc(hb, (unsigned int)ord); + if (base) { + backing = HG_LEAF_SIZE << ord; + chunk_ord = ord; + chunk_size = backing - sizeof(struct hg_large_chunk); + } + } else { + unsigned long np = (total + hb->hps - 1) >> hb->hps_shift; + + base = hg_buddy_alloc_run(hb, np); + if (base) { + backing = np << hb->hps_shift; + chunk_size = backing - sizeof(struct hg_large_chunk); + } + } + } + chunk_size &= ~63UL; + } + if (!base) { + lock_release(&hb->lock); + LM_ERR("%s: no more HG_MALLOC arena memory for a %lu byte " + "large chunk (need %lu bytes for this allocation) - " + "increase the arena size\n", hb->name, chunk_size, need); + return NULL; + } + + ch = (struct hg_large_chunk *)(void *)base; + ch->size = chunk_size; + ch->first_frag = (struct hg_lfrag *)(void *)(base + sizeof(*ch)); + ch->last_frag = (struct hg_lfrag *)(void *) + ((char *)ch->first_frag + chunk_size) - 1; + + ch->first_frag->size = chunk_size - HG_LFRAG_HDR - HG_LFRAG_HDR; + ch->first_frag->pf = NULL; + ch->first_frag->prev = NULL; + ch->last_frag->size = 0; + ch->last_frag->pf = ch->first_frag; + ch->last_frag->prev = NULL; + + ch->next = hb->large_chunks; + hb->large_chunks = ch; + ch->backing = backing; + ch->ord = chunk_ord; + hb->large_chunks_carved++; + + /* + * Charge the WHOLE buddy block the chunk sits in, the moment it + * is taken - exactly what carve_chunk() does for a slab block. + * Charging only the live fragments (which is what this used to + * do) meant the chunk's slack, and every byte of a chunk whose + * fragments had all been freed, stayed invisible: real_used + * under-reported the arena's footprint and free_to_carve + * over-reported the room left, without bound. + */ + hb->large_backing += backing; + hb->real_used += backing; + if (hb->real_used > hb->max_real_used) + hb->max_real_used = hb->real_used; + + lfrag_insert_free(hb, ch->first_frag); + f = ch->first_frag; + /* f->size is guaranteed >= need: the chunk was sized for it */ + } + + lfrag_remove_free(f); + + /* split off the remainder if it's big enough to be worth keeping as + * its own free frag (mirrors f_malloc's fm_split_frag threshold) */ + rest = f->size - need; + if (rest > HG_LFRAG_HDR + HG_LARGE_MIN_FRAG) { + f->size = need; + n = HG_LFRAG_NEXT(f); + n->size = rest - HG_LFRAG_HDR; + n->pf = f; + HG_LFRAG_NEXT(n)->pf = n; + n->prev = NULL; + lfrag_insert_free(hb, n); + } + + /* charge the frag's actual payload capacity, NOT the requested @size: + * hg_large_free() can only ever know the capacity, so charging the + * request here would leave a permanent per-allocation drift between + * the two */ + { + struct hg_pstat *ps = hg_pstat_mine(hb); + ps->used += f->size - HG_CELL_HDR; + ps->fragments++; + } + /* the chunk's backing was charged to real_used when the chunk was + * taken from the buddy; this tracks how much of it is handed out, so + * hg_large_recycled() can hand the idle remainder back to the live + * figure the way hg_slab_recycled() does for the slab tier */ + hb->large_live += HG_LFRAG_HDR + f->size; + + lock_release(&hb->lock); + + tag = (char *)f + HG_LFRAG_HDR; + *(unsigned char *)tag = HG_LARGE_MARKER; +#ifdef DBG_MALLOC + { + const char **hfile = (const char **)(tag + HG_ROUNDTO); + const char **hfunc = (const char **)(tag + HG_ROUNDTO * 2); + unsigned long *hline = (unsigned long *)(tag + HG_ROUNDTO * 3); + *hfile = file; + *hfunc = func; + *hline = line; + } +#endif + + return tag + HG_CELL_HDR; +} + +void hg_large_free(struct hg_block *hb, struct hg_lfrag *frag) +{ + struct hg_lfrag *neigh; + + lock_get(&hb->lock); + + { + struct hg_pstat *ps = hg_pstat_mine(hb); + ps->used -= frag->size - HG_CELL_HDR; + ps->fragments--; + } + hb->large_live -= HG_LFRAG_HDR + frag->size; + + /* forward coalesce - neigh->prev is NULL both for allocated frags AND + * for a chunk's sentinel, so this naturally stops at the boundary */ + neigh = HG_LFRAG_NEXT(frag); + if (neigh->prev) { + lfrag_remove_free(neigh); + frag->size += HG_LFRAG_HDR + neigh->size; + HG_LFRAG_NEXT(neigh)->pf = frag; + } + + /* backward coalesce */ + neigh = frag->pf; + if (neigh && neigh->prev) { + lfrag_remove_free(neigh); + neigh->size += HG_LFRAG_HDR + frag->size; + HG_LFRAG_NEXT(frag)->pf = neigh; + frag = neigh; + } + + /* + * Chunk empty? O(1), and it needs no search: only a chunk's first + * fragment has a NULL pf (hg_large_alloc sets it once at chunk setup + * and every split gives the remainder a non-NULL one), and only the + * chunk's sentinel has size 0. So a fragment that is BOTH first and + * followed by the sentinel spans the entire chunk - nothing else in it + * is allocated. The chunk header sits immediately in front of that + * first fragment, which is how we get back to it. + * + * Returning it unconditionally, with no keep-one hysteresis: the slab + * tier's equivalent is HG_GC_KEEP 0, and holding a chunk back is + * exactly the invisible retention this tier was just fixed for. The + * risk it trades against is alloc/free thrash on a workload that + * repeatedly empties and refills the tier, which is why both a carve + * and a return counter are exported - carved climbing far faster than + * returned is what that would look like. + */ + if (!frag->pf && HG_LFRAG_NEXT(frag)->size == 0) { + struct hg_large_chunk *ch = (struct hg_large_chunk *)(void *) + ((char *)frag - sizeof(struct hg_large_chunk)); + + if (ch->first_frag == frag) { + struct hg_large_chunk **pp; + void *base = ch; + unsigned long backing = ch->backing; + int ord = ch->ord; + + for (pp = &hb->large_chunks; *pp; pp = &(*pp)->next) + if (*pp == ch) { + *pp = ch->next; + break; + } + + hb->large_backing -= backing; + hb->real_used -= backing; + hb->large_chunks_returned++; + + /* nothing below may touch ch or frag: the buddy writes + * its free-list linkage over the first bytes of what it + * is handed */ + if (ord >= 0) + hg_buddy_free(hb, base, (unsigned int)ord); + else + hg_buddy_free_run(hb, base); + + lock_release(&hb->lock); + return; + } + } + + lfrag_insert_free(hb, frag); + + lock_release(&hb->lock); +} + +unsigned long hg_large_frag_size(const struct hg_lfrag *frag) +{ + return HG_LFRAG_HDR + frag->size; +} + +/* opaque-pointer wrapper for hg_malloc.h's hg_frag_size() - see the + * HG_LFRAG_HDR_SIZE comment there for why this indirection exists */ +unsigned long hg_large_frag_size_at(const void *frag) +{ + return hg_large_frag_size((const struct hg_lfrag *)frag); +} + +/* catches any future drift between this and HG_LFRAG_HDR_SIZE (hg_malloc.h, + * which can't sizeof() the opaque-there struct hg_lfrag directly) */ +/* the alignment guarantee this allocator hands its callers: cells and frags + * both start 8-aligned only if every step between them is a multiple of 8 */ +_Static_assert(HG_CELL_HDR % HG_PAYLOAD_ALIGN == 0, + "HG_CELL_HDR must be a multiple of HG_PAYLOAD_ALIGN or payloads misalign"); +_Static_assert(HG_LFRAG_HDR_SIZE % HG_PAYLOAD_ALIGN == 0, + "sizeof(struct hg_lfrag) must be a multiple of HG_PAYLOAD_ALIGN"); + +_Static_assert(sizeof(struct hg_lfrag) == HG_LFRAG_HDR_SIZE, + "HG_LFRAG_HDR_SIZE in hg_malloc.h must match sizeof(struct hg_lfrag)"); + +void hg_large_destroy(struct hg_block *hb) +{ + /* Chunks come from the buddy grid (hg_buddy_alloc / _alloc_run), not + * from the bump allocator as the original comment here claimed. Either + * way the whole reservation goes back in hg_malloc_destroy()'s munmap, + * so there is still nothing to release one chunk at a time - only the + * bookkeeping to drop. */ + hb->large_chunks = NULL; + hb->large_free = NULL; + hb->large_backing = 0; + hb->large_live = 0; +} + +void hg_large_walk_live(struct hg_block *hb, + void (*cb)(void *payload, void *ctx), void *ctx) +{ + struct hg_large_chunk *ch; + struct hg_lfrag *f; + + lock_get(&hb->lock); + + for (ch = hb->large_chunks; ch; ch = ch->next) + for (f = ch->first_frag; f != ch->last_frag; f = HG_LFRAG_NEXT(f)) + if (!f->prev) /* live, exact - see hg_large.h */ + cb((char *)f + HG_LFRAG_HDR + HG_CELL_HDR, ctx); + + lock_release(&hb->lock); +} + +#ifdef SHM_EXTRA_STATS +static void hg_large_stats_core_init_cb(void *payload, void *ctx) +{ + HG_STATS_IDX(payload) = (unsigned long)*(int *)ctx; +} + +void hg_large_stats_core_init(struct hg_block *hb, int core_index) +{ + hg_large_walk_live(hb, hg_large_stats_core_init_cb, &core_index); +} +#endif + +#endif /* HG_MALLOC */ diff --git a/mem/hg_large.h b/mem/hg_large.h new file mode 100644 index 00000000000..52752818b04 --- /dev/null +++ b/mem/hg_large.h @@ -0,0 +1,109 @@ +/* + * hugepage-backed slab allocator - large-object tier + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * The small-object tier (hg_arena.c) caps out at HG_CELL_MAX (64K) by + * design - fixed size classes, no splitting. Real OpenSIPS processes need + * bigger single allocations than that even at plain startup (confirmed + * empirically: init_pvar_support() alone needs ~140K), so this tier is a + * hard prerequisite for HG_MALLOC to be usable, not an optional later + * optimization. + * + * Design: boundary-tag fragments (ported from f_malloc's fm_frag/split/ + * coalesce logic), but with two deliberate simplifications given large + * allocations are inherently rare and slow-path already: + * - ONE unsorted free list per hg_block, linear first-fit search, not + * f_malloc's size-hashed buckets. Correctness and coalescing matter + * here; O(1) lookup doesn't. + * - Frags never coalesce ACROSS chunk boundaries. Growth happens by + * bump-carving an additional chunk from the SAME hoff arena the + * small-object tier uses (hg_arena.c's hg_chunk_backing()) whenever no + * existing chunk has a big-enough free frag; each chunk is internally + * a self-contained f_malloc-style heap. This mirrors how the small + * tier itself grows (one more chunk, not a bigger single arena). + * + * Dispatch: hg_cell_alloc() (hg_arena.c) routes oversized requests here + * instead of failing; hg_free()/hg_realloc() (hg_malloc_dyn.h) distinguish + * a large frag from a small cell via the class-or-marker byte at the SAME + * fixed HG_CELL_HDR offset before every payload pointer (HG_LARGE_MARKER, + * a value >= HG_NCLASSES that small-cell code already treats as "not a + * valid class" - repurposed here as "this is a large frag", not + * corruption; see hg_malloc.h HG_CLASS/HG_HDR). + */ + +#ifndef hg_large_h +#define hg_large_h + +#include "hg_malloc.h" + +struct hg_lfrag { + unsigned long size; /* total frag size: hdr + tag + payload */ + struct hg_lfrag *nxt_free; /* NULL if allocated */ + struct hg_lfrag **prev; /* NULL if allocated - f_malloc-style + * free-list back-link for O(1) removal */ + struct hg_lfrag *pf; /* physical previous frag in this chunk, + * NULL if first frag of its chunk */ +} __attribute__ ((aligned (ROUNDTO))); + +#define HG_LFRAG_HDR (sizeof(struct hg_lfrag)) + +/* + * @size is the caller's requested USABLE payload size (HG_CELL_HDR and + * HG_LFRAG_HDR are added internally). Unlike hg_cell_alloc()/hg_cell_free() + * (hg_arena.h), these have only ONE signature each, not a DBG_MALLOC-gated + * pair: hg_large.c is compiled once, always seeing DBG_MALLOC's true global + * state directly (no hg_malloc.c-style local #undef dance), so there's no + * ambiguity to guard against here. + */ +#ifdef DBG_MALLOC +void *hg_large_alloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line); +#else +void *hg_large_alloc(struct hg_block *hb, unsigned long size); +#endif + +/* @frag is the struct hg_lfrag* at the very start of the block (i.e. + * HG_HDR(payload) - HG_LFRAG_HDR), not the payload pointer itself */ +void hg_large_free(struct hg_block *hb, struct hg_lfrag *frag); + +/* total frag size (hdr+tag+payload), for hg_frag_size()/stats */ +unsigned long hg_large_frag_size(const struct hg_lfrag *frag); + +/* opaque-pointer wrapper - declared in hg_malloc.h (included above), not + * redeclared here, since that's the one callers who can't see struct + * hg_lfrag need */ + +void hg_large_destroy(struct hg_block *hb); + +/* invokes @cb(payload_ptr, ctx) for every currently-live large frag. Unlike + * hg_arena_walk_live() (small cells), this needs no auxiliary free-address + * set: a large frag's own ->prev field IS the free/live flag (NULL = live), + * exactly like f_malloc's frag_is_free(), so this is always exact - + * regardless of pre-/post-fork - not best-effort. */ +void hg_large_walk_live(struct hg_block *hb, + void (*cb)(void *payload, void *ctx), void *ctx); + +#ifdef SHM_EXTRA_STATS +void hg_large_stats_core_init(struct hg_block *hb, int core_index); +#endif + +#endif /* hg_large_h */ diff --git a/mem/hg_malloc.c b/mem/hg_malloc.c new file mode 100644 index 00000000000..8493b5650ae --- /dev/null +++ b/mem/hg_malloc.c @@ -0,0 +1,1199 @@ +/* + * hugepage-backed slab allocator + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef HG_MALLOC + +#include +#include +#include +#include +#include +#include + +#include "hg_version.h" +#include "hg_malloc.h" +#include "hg_arena.h" +#include "hg_large.h" +#include "../dprint.h" +#include "../globals.h" +#include "../statistics.h" +#include "../pt_scaling.h" /* profiles + counted_max_processes via pt.h */ +#include "shm_mem.h" /* shm_block, for the post-cfg attach */ + +#ifdef DBG_MALLOC +#include "mem_dbg_hash.h" +#endif + +#include "../lib/dbg/struct_hist.h" + +/* + * The huge-page ladder is Linux-only, and the fallback values below are + * Linux's. They are defined ONLY under __OS_linux on purpose: the same bit + * means something else elsewhere (0x40000 is MAP_PREFAULT_READ on FreeBSD), + * so defining them unconditionally would make mmap() succeed with unrelated + * semantics and this code would then report a huge-page tier it never got. + * On every other OS the reservation is a plain anonymous mapping. + */ +#ifdef __OS_linux +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0x40000 +#endif +#ifndef MADV_HUGEPAGE +#define MADV_HUGEPAGE 14 +#endif +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#endif /* __OS_linux */ + +/* + * The system's default huge page size, probed once rather than assumed. + * + * It is 2M on x86_64 and on arm64 with 4K base pages, but 32M on arm64 + * with 16K pages and 512M with 64K pages. Getting it wrong is not + * cosmetic: mmap(MAP_HUGETLB) without MAP_HUGE_* bits uses the system + * default, and the kernel rounds the mapping up to it - so a hardcoded + * 2M would (a) leave hsize describing a smaller region than the VMA, + * making the matching munmap() fail with EINVAL and leak the arena, and + * (b) align the THP tiers to 2M instead of the real PMD granularity, so + * MADV_HUGEPAGE/MADV_COLLAPSE quietly do nothing and every arena silently + * degrades to 4K while still reporting a huge-page tier. + * + * Falls back to 2M only if /proc/meminfo cannot be read at all. + */ +#define HG_HPS_FALLBACK (2UL * 1024 * 1024) + +static unsigned long hg_hps_cached; + +static unsigned long hg_hps(void) +{ + FILE *f; + char line[256]; + unsigned long kb = 0; + + if (hg_hps_cached) + return hg_hps_cached; + + f = fopen("/proc/meminfo", "r"); + if (f) { + while (fgets(line, sizeof line, f)) { + if (!strncmp(line, "Hugepagesize:", 13)) { + kb = strtoul(line + 13, NULL, 10); + break; + } + } + fclose(f); + } + + /* must be a power of two for the alignment masks below to work */ + if (kb == 0 || (kb * 1024UL) & ((kb * 1024UL) - 1)) + hg_hps_cached = HG_HPS_FALLBACK; + else + hg_hps_cached = kb * 1024UL; + + return hg_hps_cached; +} + +#define HG_HPS (hg_hps()) + +/* round @s up to a whole number of huge pages. Used for the reservation + * length, the alignment of the THP tiers, AND the matching munmap length - + * they must agree exactly or the unmap fails and the arena leaks, so they + * all go through this one macro rather than repeating the expression. */ +#define HG_HPS_ROUND(s) (((s) + HG_HPS - 1) & ~(HG_HPS - 1)) + +/* HG_ROUNDTO=2^k so the following works (same trick as f_malloc.c) */ +#define ROUNDTO_MASK (~((unsigned long)HG_ROUNDTO-1)) +#define ROUNDUP_TO(s) (((s)+(HG_ROUNDTO-1))&ROUNDTO_MASK) + +/* + * Tier ladder + verification, ported near-verbatim from cachedb_perf's + * pcache_mem.c pcache_mem_reserve() (DESIGN 2.6.1/2.6.2 there): every tier + * is proven by TRYING it and verifying the result through /proc, never + * inferred from kernel version or sysfs config alone. + */ + +/* read one "Key: kB" line; -1 if absent (old kernel, no /proc) */ +static long hg_meminfo_kb(const char *key) +{ + FILE *f; + char line[256]; + size_t klen = strlen(key); + long kb = -1; + + f = fopen("/proc/meminfo", "r"); + if (!f) + return -1; + while (fgets(line, sizeof line, f)) { + if (!strncmp(line, key, klen) && line[klen] == ':') { + kb = strtol(line + klen + 1, NULL, 10); + break; + } + } + fclose(f); + return kb; +} + +static long hg_read_shmem_huge_kb(void) +{ + return hg_meminfo_kb("ShmemHugePages"); +} + +/* is the huge-page-sized range starting at @addr PMD-mapped here? */ +static int hg_range_is_huge(unsigned long addr) +{ + FILE *f; + char line[256], *p; + unsigned long start, end, kb; + int in_range = 0, huge = 0; + + f = fopen("/proc/self/smaps", "r"); + if (!f) + return 0; + + while (fgets(line, sizeof line, f)) { + if (sscanf(line, "%lx-%lx ", &start, &end) == 2) { + in_range = (start <= addr && addr < end); + continue; + } + if (!in_range) + continue; + if (!strncmp(line, "AnonHugePages:", 14) || + !strncmp(line, "ShmemPmdMapped:", 15) || + !strncmp(line, "FilePmdMapped:", 14)) { + p = strchr(line, ':'); + kb = strtoul(p + 1, NULL, 10); + if (kb >= HG_HPS / 1024) { + huge = 1; + break; + } + } + } + + fclose(f); + return huge; +} + +/* + * Keep the arena out of core dumps - unless someone is trying to debug the + * allocator, in which case the arena is the only thing worth having. + * + * HG_MALLOC pre-faults and mlocks its whole reservation, so unlike the + * lazily-faulted F_MALLOC/Q_MALLOC pools every page is resident - a + * crashing worker would otherwise write the ENTIRE arena (-m plus -M, and + * the shm_memlog_size-derived debug pool on top) into its core file. With + * one core per worker that is multi-GB of core dumps per crash, which is + * enough page-cache churn to push a busy box into reclaim, and it buries + * the actually-useful stack/heap in gigabytes of allocator slab. + * + * The cost of that default only became clear when a core was actually needed: + * VM_DONTDUMP wins over coredump_filter, so no filter setting can bring the + * arena back, and every core taken during the 2026-08 crash investigation had + * "Cannot access memory" where shm_block should be. The free lists, the class + * counters, the cell headers - the entire state that decides whether a crash + * was corruption or a race - are all inside the region being skipped. + * + * So it is opt-in: set HG_DUMP_ARENA=1 in the environment (a systemd + * Environment= line is enough) and the arena is dumped. Sized deliberately as + * an environment variable rather than a config parameter, because it must take + * effect during allocator init, long before the config file is parsed. + * + * Best-effort: MADV_DONTDUMP is Linux 3.4+, and failure is harmless + * (bigger cores, nothing incorrect), so the return value is ignored. + */ +static void hg_exclude_from_core(void *base, unsigned long size) +{ +#ifdef MADV_DONTDUMP + const char *want = getenv("HG_DUMP_ARENA"); + + if (want && *want && *want != '0') { + /* explicit, not merely "leave the default alone" - the mapping + * may have inherited VM_DONTDUMP from a previous madvise on an + * overlapping range */ + madvise(base, size, MADV_DODUMP); + return; + } + + madvise(base, size, MADV_DONTDUMP); +#endif +} + +/* + * Reserve a huge-page-aligned, huge-page-backed (best effort) region of at least + * @size bytes, mlock-pinned against swap. Never unmapped until + * hg_malloc_destroy(). Returns NULL on total mmap failure only - a huge-page + * miss still returns a valid plain-4K mapping (degrade, don't fail), per + * hg_mem_tier_str()'s HG_MEM_4K case. + * + * @shared picks MAP_SHARED vs MAP_PRIVATE, and it is NOT cosmetic: + * + * shm -> MAP_SHARED: one arena visible to every forked worker, which is + * the entire point of shm. + * pkg -> MAP_PRIVATE: every worker must get its OWN copy-on-write arena + * after fork. Mapping the pkg arena MAP_SHARED (as + * this function originally did unconditionally) put + * the pkg hg_block - including its embedded + * gen_lock_t and its per-class gpool free lists - in + * memory shared by all workers. Under FAST_LOCK that + * lock is a *spinlock*, so every worker's pkg + * allocations serialized on one contended spinlock, + * and "process-private" pkg cells silently migrated + * between processes through the shared gpool. + */ +static void *hg_mem_reserve(unsigned long size, unsigned long *cap, + enum hg_mem_tier *tier, unsigned long *locked_mb, int shared, + int inherited) +{ + unsigned long asize = HG_HPS_ROUND(size); + unsigned long csize = HG_HPS_ROUND(*cap < size ? size : *cap); + int vis = shared ? MAP_SHARED : MAP_PRIVATE; + char *resv, *base; + long shmem_kb; + void *p; + + *locked_mb = 0; + *tier = HG_MEM_4K; + *cap = csize; /* rewritten below if a fallback shrinks it */ + +#ifndef __OS_linux + /* No verified huge-page route outside Linux: take a plain anonymous + * mapping and report the 4K tier honestly rather than claiming one we + * cannot check. Still pinned and pre-faulted. */ + p = mmap(NULL, csize, PROT_READ|PROT_WRITE, vis|MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return NULL; + hg_exclude_from_core(p, csize); + if (mlock(p, asize) == 0) + *locked_mb = asize >> 20; + else + memset(p, 0, asize); + return p; +#else + + /* + * tier 1: MAP_HUGETLB - unswappable, exempt from RLIMIT_MEMLOCK. + * + * Try the whole cap first, then fall back to the committed size alone. + * hugetlb mappings are backed by a fixed pool, so a cap larger than the + * pool can hold makes this mmap fail outright - and silently dropping + * to THP because the admin asked for growth room would be a far worse + * trade than simply not being able to grow. A cap-less tier-1 arena is + * what v2 shipped; losing the tier is a real regression. + */ + /* + * ...except for the arena children inherit copy-on-write (the pre-fork + * pkg arena, HG_INIT_INHERITED): a child's write into an inherited + * hugetlb page needs a fresh huge page with no 4K fallback and no + * reservation behind it - an empty pool at that instant is a SIGBUS. + * That arena starts the ladder at THP, whose COW splits to 4K pages + * instead. See the flag's comment in hg_malloc.h. + */ + p = inherited ? MAP_FAILED : mmap(NULL, csize, PROT_READ|PROT_WRITE, + vis|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); + if (p == MAP_FAILED && !inherited && csize > asize) { + p = mmap(NULL, asize, PROT_READ|PROT_WRITE, + vis|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + LM_NOTICE("hugetlb pool cannot back a %lu MB cap; " + "reserving the %lu MB in use instead - the arena " + "keeps huge pages but cannot grow. Raise " + "vm.nr_hugepages to allow growth.\n", + csize >> 20, asize >> 20); + *cap = asize; /* the arena is fixed after all */ + } + /* on total failure csize stays at the full cap for tiers 2-4: + * THP reservations are plain VA, which CAN hold the cap */ + } + if (p != MAP_FAILED) { + hg_exclude_from_core(p, *cap); + memset(p, 0, asize); + *tier = HG_MEM_HUGETLB; + *locked_mb = asize >> 20; + return p; + } + + /* + * tiers 2-4: huge-page-aligned anon mapping. For the shmem + * (MAP_SHARED) case the VA and shmem *file offset* must be congruent + * modulo the huge page size for THP eligibility, so reserve PROT_NONE + * first, then MAP_FIXED the real mapping at a huge-page boundary + * inside it - an atomic replace, no race with other mappings. + * Harmless (and keeps the alignment) for MAP_PRIVATE. + * + * The real mapping covers the whole CAP, readable and writable, even + * though only asize of it is committed now. That is the load-bearing + * part of v3 growth, not an accident: this mapping is created before + * fork, so it is the one VMA every worker inherits, all of them backed + * by the same shmem object. Growing later means faulting more of that + * object in - visible to every process by construction. The obvious + * alternative - keep the tail PROT_NONE and mmap/mprotect it live at + * grow time - changes only the GROWER's page tables: measured on the + * 5.4 kernel, the grower reads its new pages fine and a forked worker + * SIGSEGVs on the same addresses (scratchpad rig vatest.c, test A vs + * B). An untouched R/W tail costs a few hundred kB of page-table + * entries, not memory - test B: 64 MB of mapped-untouched span held + * RSS at 576 kB. + */ + resv = mmap(NULL, csize + HG_HPS, PROT_NONE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (resv == MAP_FAILED) + return NULL; + base = (char *)(((unsigned long)resv + HG_HPS - 1) & ~(HG_HPS - 1)); + p = mmap(base, csize, PROT_READ|PROT_WRITE, + vis|MAP_ANONYMOUS|MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + munmap(resv, csize + HG_HPS); + return NULL; + } + + hg_exclude_from_core(base, csize); + + /* advise huge before first touch (tier 2), then pin+populate: a cold + * mlock populates to pin, so it doubles as the pre-fault. The advice + * covers the whole cap so growth deltas inherit it - each delta still + * gets its backing VERIFIED at grow time, never assumed from here. */ + madvise(base, csize, MADV_HUGEPAGE); + shmem_kb = hg_read_shmem_huge_kb(); + if (mlock(base, asize) == 0) { + *locked_mb = asize >> 20; + } else { + LM_WARN("mlock of the %lu MB HG_MALLOC arena failed (%s): " + "continuing unpinned (swappable). If running under " + "systemd, add LimitMEMLOCK=infinity to the unit.\n", + asize >> 20, strerror(errno)); + memset(base, 0, asize); /* still pre-fault */ + } + + if (hg_range_is_huge((unsigned long)base)) { + *tier = HG_MEM_THP_ADVISE; + } else if (shmem_kb >= 0 && + madvise(base, asize, MADV_COLLAPSE) == 0 && + hg_read_shmem_huge_kb() - shmem_kb >= (long)(asize / 1024)) { + *tier = HG_MEM_THP_COLLAPSE; + } else { + *tier = HG_MEM_4K; /* reserved+pinned but 4K */ + } + return base; +#endif /* __OS_linux */ +} + +/* + * The host-RAM limb of the growth ceiling - see the prototype comment. + * + * The floor it defends is max(256 MB, MemTotal/20), overridable via the + * hg_ram_floor_mb config global. + * MemAvailable is the kernel's own estimate of what can be claimed + * without swapping - exactly the question here. On a kernel too old to + * export it the check PASSES: the mlock in hg_mem_commit() still refuses + * with a clean errno when the host truly cannot back the delta, so the + * failure mode without this limb is a later, harsher refusal, not a + * crash. + * + * Reading /proc under hb->lock is deliberate, same trade as the commit + * pre-fault: growth is once per granule of genuine demand, and the + * mlock that follows costs orders of magnitude more than one procfs + * read. + */ +int hg_grow_ram_refused(struct hg_block *hb, unsigned long delta) +{ + static long floor_mb = -1; /* resolved once, per process */ + unsigned long effective = delta, nproc = 1; + long avail_kb; + + /* tier 1 consumes no new host RAM at commit time: the whole cap was + * reserved from the hugetlb pool at map time, and those pages are + * already carved out of MemTotal. Charging them here double-counts. */ + if (hb->tier == HG_MEM_HUGETLB) + return 0; + + if (floor_mb < 0) { + if (hg_ram_floor_mb > 0) { + floor_mb = hg_ram_floor_mb; /* hg_ram_floor_mb= config */ + } else { + long total_kb = hg_meminfo_kb("MemTotal"); + + floor_mb = 256; + if (total_kb > 0 && total_kb / 20 / 1024 > floor_mb) + floor_mb = total_kb / 20 / 1024; + } + } + + avail_kb = hg_meminfo_kb("MemAvailable"); + if (avail_kb < 0) + return 0; /* cannot tell - let mlock decide */ + + if (!hb->shared) { + /* pkg: every worker will grow its own arena under the same + * workload; the single-arena delta understates the real cost + * by the process count */ + nproc = counted_max_processes ? counted_max_processes : 1; + effective = delta * nproc; + } + + if ((unsigned long)avail_kb * 1024 < + effective + ((unsigned long)floor_mb << 20)) { + /* once per episode - see grow_refuse_said's comment */ + if (!hb->grow_refuse_said) { + hb->grow_refuse_said = 1; + LM_WARN("%s: refusing to grow by %lu MB: %lu MB effective" + " (x%lu processes) would leave the host under the " + "%ld MB floor (MemAvailable %ld MB). Freeing host " + "memory or lowering the floor lifts this.\n", + hb->name, delta >> 20, effective >> 20, nproc, + floor_mb, avail_kb / 1024); + } + return 1; + } + return 0; +} + +/* + * Commit [hbase+off, +delta) of the reservation: populate, pin, verify the + * achieved backing. The range is already mapped R/W (the whole cap is, since + * reserve time - that is what makes the commit visible to every forked + * worker with no page-table surgery here), so the only work is faulting the + * pages in and finding out what the kernel faulted them in AS. + * + * mlock() is the commit primitive for every tier, chosen for one property: + * it populates the exact range and reports failure through errno instead of + * raising SIGBUS in whichever worker touches the shortfall later. A grow + * that cannot be backed must fail HERE, atomically, while the buddy still + * considers the range nonexistent. + * - tiers 2-4: mlock is also the pin, same as init. + * - tier 1: hugetlb pages are unswappable regardless; mlock is used only + * as the populate-with-clean-errno vehicle. The pool-exhaustion path + * (mlock ENOMEM, nothing SIGBUSes, VM_LOCKED rolled back) is PROVEN by + * the hgstress grow harness against a deliberately undersized pool - + * do not take this comment's word for it, run the harness. + * + * Returns the achieved hg_mem_tier of the delta, or -1 with the range + * munlock'd again (refuse, never half-commit). No hg_exclude_from_core() + * here: reserve time already excluded the whole cap. + */ +int hg_mem_commit(struct hg_block *hb, unsigned long off, unsigned long delta) +{ + char *base = hb->hbase + off; + + if (off + delta > hb->hcap) { + LM_BUG("%s: commit of %lu@%lu overruns the %lu byte cap\n", + hb->name, delta, off, hb->hcap); + return -1; + } + + if (hb->tier == HG_MEM_HUGETLB) { + if (mlock(base, delta) != 0) { + /* once per episode - see grow_refuse_said's comment */ + if (!hb->grow_refuse_said) { + hb->grow_refuse_said = 1; + LM_WARN("%s: cannot grow by %lu MB: the hugetlb " + "pool is exhausted (%s). Raise " + "vm.nr_hugepages.\n", + hb->name, delta >> 20, strerror(errno)); + } + munlock(base, delta); + return -1; + } + hb->locked_mb += delta >> 20; + return HG_MEM_HUGETLB; + } + +#ifdef __OS_linux + { + long shmem_kb = hg_read_shmem_huge_kb(); + + /* re-advise the delta: cheap, and correct even though reserve + * time advised the whole cap - a later madvise elsewhere in the + * VMA may have split it */ + madvise(base, delta, MADV_HUGEPAGE); + + if (mlock(base, delta) != 0) { + /* once per episode - see grow_refuse_said's comment */ + if (!hb->grow_refuse_said) { + hb->grow_refuse_said = 1; + LM_WARN("%s: cannot grow by %lu MB: mlock failed " + "(%s). If running under systemd, add " + "LimitMEMLOCK=infinity to the unit.\n", + hb->name, delta >> 20, strerror(errno)); + } + munlock(base, delta); + return -1; + } + hb->locked_mb += delta >> 20; + + /* + * The delta's backing is a fresh negotiation - the arena's init + * tier says NOTHING about what this range just got. Verify it + * the same way init does: read what the kernel actually did. + */ + if (hg_range_is_huge((unsigned long)base)) + return HG_MEM_THP_ADVISE; + if (shmem_kb >= 0 && + madvise(base, delta, MADV_COLLAPSE) == 0 && + hg_read_shmem_huge_kb() - shmem_kb >= (long)(delta / 1024)) + return HG_MEM_THP_COLLAPSE; + return HG_MEM_4K; + } +#else + if (mlock(base, delta) != 0) { + memset(base, 0, delta); /* still pre-fault */ + } else { + hb->locked_mb += delta >> 20; + } + return HG_MEM_4K; +#endif +} + +/* the shrink primitive - contract and measurements on the prototype */ +int hg_mem_release(struct hg_block *hb, unsigned long off, unsigned long len) +{ + char *base = hb->hbase + off; + int advice = hb->shared ? MADV_REMOVE : MADV_DONTNEED; + + if (off + len > hb->hsize) { + LM_BUG("%s: release of %lu@%lu overruns the %lu committed " + "bytes\n", hb->name, len, off, hb->hsize); + return -1; + } + + munlock(base, len); + + if (madvise(base, len, advice) != 0) { + /* structural, not transient: the advice either works on this + * mapping type + kernel or it never will. Say so once and stop + * trying for this arena's lifetime. */ + hb->shrink_unsupported = 1; + LM_WARN("%s: cannot release memory (%s of %lu MB failed: %s) " + "- shrink disabled for this arena\n", hb->name, + hb->shared ? "MADV_REMOVE" : "MADV_DONTNEED", + len >> 20, strerror(errno)); + return -1; + } + + if (hb->locked_mb >= len >> 20) + hb->locked_mb -= len >> 20; + else + hb->locked_mb = 0; + return 0; +} + +/* the pkg policy, resolved once post-parse and inherited by fork - every + * per-child arena pt.c creates copies it in at hg_malloc_init() time */ +static struct { + int valid; + unsigned long up_bytes, down_bytes; + unsigned int up_pct, up_need, up_window, down_pct, down_cycles; + unsigned short cooldown; +} hg_pkg_pol_resolved; + +/* + * Copy a profile's numbers onto an arena, translating workers->MB and + * validating every edge against the reservation this arena actually has. + * @hb may be NULL for the pkg case (arena does not exist yet) - then only + * the validation against @cap/@init runs and the result lands in + * hg_pkg_pol_resolved. + */ +static int hg_autoscale_apply(struct hg_block *hb, const char *which, + struct scaling_profile *p, unsigned long init_bytes, + unsigned long cap_bytes) +{ + unsigned long up_b = HG_HPS_ROUND((unsigned long)p->max_procs << 20); + unsigned long down_b = p->min_procs + ? HG_HPS_ROUND((unsigned long)p->min_procs << 20) : 0; + + if (cap_bytes <= HG_HPS_ROUND(init_bytes)) { + LM_ERR("%s profile '%s': the arena has no growth room - give " + "the reservation on the command line (-%s INIT:CAP)\n", + which, p->name, hb || !strcmp(which, "shm") ? "m" : "M"); + return -1; + } + if (up_b <= HG_HPS_ROUND(init_bytes)) { + LM_ERR("%s profile '%s': scale-up target %u MB does not exceed " + "the initial %lu MB - the profile could never act\n", + which, p->name, p->max_procs, init_bytes >> 20); + return -1; + } + if (up_b > cap_bytes) { + LM_ERR("%s profile '%s': scale-up target %u MB exceeds the " + "%lu MB reservation - raise the :CAP\n", + which, p->name, p->max_procs, cap_bytes >> 20); + return -1; + } + if (down_b) { + if (down_b < 2 * HG_HPS) { + LM_ERR("%s profile '%s': scale-down target %u MB is below " + "the %lu MB minimum viable arena\n", + which, p->name, p->min_procs, (2 * HG_HPS) >> 20); + return -1; + } + if (down_b >= up_b) { + LM_ERR("%s profile '%s': scale-down target %u MB is not " + "below the scale-up target %u MB\n", + which, p->name, p->min_procs, p->max_procs); + return -1; + } + } + + if (hb) { + lock_get(&hb->lock); + hb->pol.active = 1; + hb->pol.up_bytes = up_b; + hb->pol.down_bytes = down_b ? down_b : hb->hsize_min; + hb->pol.up_pct = p->up_threshold; + hb->pol.up_need = p->up_cycles_needed; + hb->pol.up_window = p->up_cycles_tocheck; + hb->pol.down_pct = p->down_threshold; + hb->pol.down_cycles = p->down_cycles_tocheck; + hb->pol.cooldown = p->down_cycles_delay; + if (down_b) + hb->hsize_min = down_b; /* the profile IS the ask now */ + lock_release(&hb->lock); + } else { + hg_pkg_pol_resolved.valid = 1; + hg_pkg_pol_resolved.up_bytes = up_b; + hg_pkg_pol_resolved.down_bytes = down_b; + hg_pkg_pol_resolved.up_pct = p->up_threshold; + hg_pkg_pol_resolved.up_need = p->up_cycles_needed; + hg_pkg_pol_resolved.up_window = p->up_cycles_tocheck; + hg_pkg_pol_resolved.down_pct = p->down_threshold; + hg_pkg_pol_resolved.down_cycles = p->down_cycles_tocheck; + hg_pkg_pol_resolved.cooldown = p->down_cycles_delay; + } + + LM_NOTICE("%s auto-scaling profile '%s'%s: %lu..%lu MB (start %lu), " + "up at %u%% for %u/%u cycles, down at %u%% for %u cycles " + "(cooldown %u)\n", which, p->name, + hg_autoscale_dry_run ? " [DRY RUN - advise only]" : "", + (down_b ? down_b : HG_HPS_ROUND(init_bytes)) >> 20, up_b >> 20, + init_bytes >> 20, p->up_threshold, p->up_cycles_needed, + p->up_cycles_tocheck, p->down_threshold, p->down_cycles_tocheck, + p->down_cycles_delay); + return 0; +} + +int hg_autoscale_post_cfg(void) +{ + struct scaling_profile *p; + int hg_shm = (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG); + int hg_pkg = (mem_allocator_pkg == MM_HG_MALLOC || + mem_allocator_pkg == MM_HG_MALLOC_DBG); + + if (hg_shm_profile_name) { + if (!hg_shm) { + LM_WARN("shm_auto_scaling_profile ignored: the shm " + "allocator is %s, not " HG_MALLOC_NAME "\n", + mm_str(mem_allocator_shm)); + } else { + p = get_scaling_profile(hg_shm_profile_name); + if (!p) { + LM_ERR("shm_auto_scaling_profile '%s' does not name " + "an auto_scaling_profile\n", hg_shm_profile_name); + return -1; + } + if (hg_autoscale_apply((struct hg_block *)shm_block, "shm", + p, shm_mem_size, hg_shm_cap_bytes) < 0) + return -1; + } + } + + if (hg_pkg_profile_name) { + if (!hg_pkg) { + LM_WARN("pkg_auto_scaling_profile ignored: the pkg " + "allocator is %s, not " HG_MALLOC_NAME "\n", + mm_str(mem_allocator_pkg)); + } else { + p = get_scaling_profile(hg_pkg_profile_name); + if (!p) { + LM_ERR("pkg_auto_scaling_profile '%s' does not name " + "an auto_scaling_profile\n", hg_pkg_profile_name); + return -1; + } + if (hg_autoscale_apply(NULL, "pkg", p, pkg_mem_size, + hg_pkg_cap_bytes) < 0) + return -1; + } + } + return 0; +} + +const char *hg_mem_tier_str(enum hg_mem_tier tier) +{ + switch (tier) { + case HG_MEM_HUGETLB: + return "MAP_HUGETLB 2M pages"; + case HG_MEM_THP_ADVISE: + return "THP 2M pages via MADV_HUGEPAGE (huge at fault)"; + case HG_MEM_THP_COLLAPSE: + return "THP 2M pages via MADV_COLLAPSE (post-fill retrofit)"; + case HG_MEM_4K: + return "plain 4K pages"; + } + return "unknown"; +} + +/* + * Registry of live arena ranges, for the ownership tests in hg_malloc.h. + * + * hg_owns_any() needs to answer "is this pointer from ANY of our arenas?" + * without being handed a block, because hg_frag_size() is installed into a + * shared function-pointer interface whose signature we do not control. A + * fixed-size array is deliberate: this is bootstrap bookkeeping for the + * allocator itself, so it must not be allocated THROUGH the allocator. + * + * Process-local, and correct under fork by construction: a child inherits the + * parent's entries (its shm mapping is genuinely the same memory) and adds its + * own private pkg arena when pt.c swaps one in. Keeping the parent's stale pkg + * entry is a feature here - a parent-allocated pkg pointer freed in a child + * still resolves to mapped memory, so it is declined rather than dereferenced. + */ +struct hg_arena_range hg_arena_reg[HG_ARENA_REG_MAX]; + +/* + * How many frees this process redirected to an arena other than the one the + * caller named. Not an error count - see hg_owner(). It is expected to be a + * small constant per child, set at startup and never moving again; a figure + * that climbs with traffic would mean something is handing pointers across + * arenas at runtime, which nothing should. + */ +unsigned long hg_xarena_frees; + +static void hg_arena_reg_add(struct hg_block *hb) +{ + int i; + + for (i = 0; i < HG_ARENA_REG_MAX; i++) { + if (!hg_arena_reg[i].base) { + hg_arena_reg[i].base = hb->hbase; + /* the CAP, not hsize: growth must not invalidate the + * registry entry, or a pointer into grown space would be + * misread as foreign and "routed" to another arena. The + * whole cap's VA belongs to this arena from reserve time; + * uncommitted ranges cannot hold live cells, so the wider + * range cannot misattribute anything that exists. */ + hg_arena_reg[i].size = hb->hcap; + hg_arena_reg[i].hb = hb; + return; + } + } + /* Not fatal: hg_owns_any() then declines pointers it cannot vouch for, + * which costs diagnostics, never correctness. */ + LM_WARN("%s: more than %d live HG_MALLOC arenas in one process - " + "ownership checks will be incomplete\n", hb->name, + HG_ARENA_REG_MAX); +} + +static void hg_arena_reg_del(struct hg_block *hb) +{ + int i; + + for (i = 0; i < HG_ARENA_REG_MAX; i++) { + if (hg_arena_reg[i].base == hb->hbase) { + hg_arena_reg[i].base = NULL; + hg_arena_reg[i].size = 0; + hg_arena_reg[i].hb = NULL; + return; + } + } +} + +/* + * hg_malloc_init() reserves its own memory (unlike fm_malloc_init(), which + * receives an already-mmap'd address from shm_getmem()) and lays the block + * control structure out at the very start of that reservation - the same + * "control struct lives inside the memory it manages" pattern fm_block/ + * hp_block use, chosen specifically so HG_MALLOC never needs to call + * shm_malloc()/pkg_malloc() on itself to bootstrap its own bookkeeping + * (it cannot: HG_MALLOC IS what those macros dispatch to when selected). + */ +struct hg_block *hg_malloc_init(unsigned long size, char *name, int shared, + const char *proc_desc, unsigned int flags) +{ + enum hg_mem_tier tier; + unsigned long locked_mb; + unsigned long cap; + char *base; + struct hg_block *hb; + + /* v3: the reservation may exceed the committed size, by admin cap. + * cap comes back as what was actually achieved (a hugetlb pool that + * cannot hold the cap degrades to a fixed arena, not to no arena). + * By NAME, not by @shared: shm_dbg is also shared but is a fixed-size + * diagnostic pool computed by hg_get_dbg_pool_size() - handing it the + * shm cap would reserve gigabytes of VA for a pool that must never + * grow past its formula. + * + * The caps arrive via -m INIT:CAP / -M INIT:CAP on the command line - + * they cannot come from the config, which is parsed only after this + * arena exists (and, for tier 1, after the pool reservation is + * already taken). */ + if (!strcmp(name, "shm")) + cap = hg_shm_cap_bytes; + else if (!strcmp(name, "pkg")) + cap = hg_pkg_cap_bytes; + else + cap = 0; + base = hg_mem_reserve(size, &cap, &tier, &locked_mb, shared, + (flags & HG_INIT_INHERITED) != 0); + if (!base) { + LM_ERR("failed to reserve %lu bytes for %s HG_MALLOC arena\n", + size, name); + return NULL; + } + + /* the block header itself lives inside the reservation it describes */ + if (size < ROUNDUP_TO(sizeof(struct hg_block))) { + LM_ERR("%s arena of %lu bytes too small for the block header " + "(%zu bytes)\n", name, size, sizeof(struct hg_block)); + munmap(base, cap); + return NULL; + } + + hb = (struct hg_block *)(void *)base; + memset(hb, 0, sizeof *hb); + hb->name = name; + hb->size = size; + hb->lo = ~0UL; + hb->hbase = base; + hb->hsize = HG_HPS_ROUND(size); + hb->hsize_min = hb->hsize; + hb->hcap = cap; + /* one committed-size step per grow: big enough that a growth spurt is + * a handful of commits, small enough that the pre-fault under the + * arena lock stays bounded. Overridable by config later. */ + hb->grow_granule = HG_HPS_ROUND(16UL << 20); + /* hg_hps() is private to this file, and hg_arena_init() needs the probed + * value to lay out the page grid - hand it over rather than re-probing */ + hb->hps = HG_HPS; + hb->tier = tier; + hb->locked_mb = locked_mb; + hb->tier_bytes[tier] = hb->hsize; + hb->shared = shared; + + /* + * A per-child PKG arena created after the config was parsed inherits + * the resolved pkg policy (the fork copied hg_pkg_pol_resolved). The + * pre-fork parent pkg arena and the shm arena take the other path: + * they exist BEFORE the config, so the shm policy is attached to the + * live block by hg_autoscale_post_cfg() and the parent pkg arena + * simply stays fixed. + */ + if (!shared && hg_pkg_pol_resolved.valid) { + hb->pol.active = 1; + hb->pol.up_bytes = hg_pkg_pol_resolved.up_bytes; + hb->pol.down_bytes = hg_pkg_pol_resolved.down_bytes + ? hg_pkg_pol_resolved.down_bytes : hb->hsize_min; + hb->pol.up_pct = hg_pkg_pol_resolved.up_pct; + hb->pol.up_need = hg_pkg_pol_resolved.up_need; + hb->pol.up_window = hg_pkg_pol_resolved.up_window; + hb->pol.down_pct = hg_pkg_pol_resolved.down_pct; + hb->pol.down_cycles = hg_pkg_pol_resolved.down_cycles; + hb->pol.cooldown = hg_pkg_pol_resolved.cooldown; + if (hg_pkg_pol_resolved.down_bytes) + hb->hsize_min = hg_pkg_pol_resolved.down_bytes; + } + + if (!lock_init(&hb->lock)) { + LM_ERR("failed to init the %s arena lock\n", name); + munmap(base, hb->hcap); + return NULL; + } + + hg_arena_reg_add(hb); + + /* the region right after the block header is the first thing chunks + * bump-carve from - hg_arena_init() sets hoff past it */ + if (hg_arena_init(hb, ROUNDUP_TO(sizeof(struct hg_block))) < 0) { + LM_ERR("failed to init the %s arena\n", name); + lock_destroy(&hb->lock); + munmap(base, hb->hcap); + return NULL; + } + + /* "pinned from swapping" is the real guarantee this reports: tier-1 + * MAP_HUGETLB pages are non-swappable by construction (no mlock() + * needed or taken), tiers 2-4 rely on an explicit mlock() instead - + * either way, the reported MB are equally protected against swap, + * just via a different mechanism. Plain "pinned" reads ambiguously + * (looks like "an mlock() call happened") and was caught live during + * a real diagnosis session mid-2026-08-07 being misread that way. */ + if (proc_desc) + LM_NOTICE("%s " HG_MALLOC_NAME " arena (%s): %lu MB on %s, %lu MB " + "pinned from swapping\n", + name, proc_desc, size >> 20, hg_mem_tier_str(tier), locked_mb); + else + LM_NOTICE("%s " HG_MALLOC_NAME " arena: %lu MB on %s, %lu MB " + "pinned from swapping%s\n", + name, size >> 20, hg_mem_tier_str(tier), locked_mb, + (flags & HG_INIT_INHERITED) ? + " (pre-fork arena, inherited copy-on-write by every child: " + "hugetlb deliberately skipped, its COW cannot fall back)" : ""); + if (hb->hcap > hb->hsize) + LM_NOTICE("%s arena can grow to %lu MB (%lu MB headroom " + "reserved, uncommitted)\n", name, hb->hcap >> 20, + (hb->hcap - hb->hsize) >> 20); + + return hb; +} + +/* mirrors fm_get_dbg_pool_size()'s structure, HG_CELL_HDR substituted for + * FRAG_OVERHEAD - see the "why" note on the declaration in hg_malloc.h */ +unsigned long hg_get_dbg_pool_size(unsigned int hist_size) +{ + return ROUNDUP_TO(sizeof(struct hg_block)) + HG_CELL_HDR + + HG_CELL_HDR + 56 /* sizeof(struct struct_hist_list) */ + 2 * hist_size * + (HG_CELL_HDR + 88 /* sizeof(struct struct_hist) */ + + HG_CELL_HDR + sizeof(struct struct_hist_action)); +} + +void hg_malloc_destroy(struct hg_block *hb) +{ + if (!hb) + return; + + hg_arena_reg_del(hb); + hg_arena_destroy(hb); + lock_destroy(&hb->lock); + /* munmap last: hb itself lives inside hbase. The whole cap, not just + * the committed part - the reservation is one mapping */ + munmap(hb->hbase, hb->hcap); +} + +void hg_malloc_child_init(struct hg_block *hb) +{ + if (hb) + hg_arena_child_init(hb); +} + +#ifdef SHM_EXTRA_STATS +#include "module_info.h" +unsigned long hg_stats_get_index(void *ptr) +{ + if (!ptr) + return GROUP_IDX_INVALID; + + return HG_STATS_IDX(ptr); +} + +void hg_stats_set_index(void *ptr, unsigned long idx) +{ + if (!ptr) + return; + + HG_STATS_IDX(ptr) = idx; +} + +/* called once, pre-fork, after the statistics engine but before any worker + * exists yet (matches hp_init_shm_statistics()'s call site) - single + * process at this point, so walking "not on any known free list" is exact, + * unlike hg_status_dbg()'s post-fork best-effort walk (see hg_arena.c's + * hg_arena_walk_live() comment) */ +void hg_stats_core_init(struct hg_block *hb, int core_index) +{ + hg_arena_stats_core_init(hb, core_index); +} +#endif + +/* fills a malloc info structure with info about the block */ +void hg_info(struct hg_block *hb, struct mem_info *info) +{ + unsigned long recycled; + + memset(info, 0, sizeof *info); + info->total_size = hb->size; + info->min_frag = 64; /* smallest cell class, see hg_arena.c cell_sizes */ + info->used = hg_used(hb); + + /* Report carved-but-idle cell capacity as FREE rather than USED, so + * real_used/free track live demand and fall again when load drops - + * the same thing q_malloc/f_malloc do for a fragment sitting on a free + * list. hb->real_used on its own is the arena's carve footprint, which + * never decreases and would otherwise look like a leak. */ + recycled = hg_slab_recycled(hb); + info->real_used = hb->real_used > recycled ? hb->real_used - recycled : 0; + /* room left to CARVE, not size minus live - see hg_get_free() */ + info->free = hb->size - hb->real_used; + + /* the peak of what real_used above actually reached - NOT the peak + * carve (hb->max_real_used), which only ever grows and would drift + * away from real_used forever */ + if (info->real_used > hb->max_live_used) + hb->max_live_used = info->real_used; + info->max_used = hb->max_live_used; + info->total_frags = hg_fragments(hb); +} + +void hg_status(struct hg_block *hb) +{ + unsigned int nchunks; + unsigned long bytes; + + LM_GEN1(memdump, "hg_status (%p):\n", hb); + if (!hb) + return; + + hg_arena_stats(hb, &nchunks, &bytes); + LM_GEN1(memdump, " heap size= %lu, tier=%s, pinned=%lu MB\n", + hb->size, hg_mem_tier_str(hb->tier), hb->locked_mb); + LM_GEN1(memdump, " chunks= %u, chunk bytes= %lu\n", nchunks, bytes); +} + +#if !defined INLINE_ALLOC && defined DBG_MALLOC +struct hg_dbg_dump_ctx { + mem_dbg_htable_t *allocd; + unsigned long skipped_notlive; /* see hg_dbg_dump_cb() */ +}; + +static void hg_dbg_dump_cb(void *payload, void *ctx) +{ + struct hg_dbg_dump_ctx *c = ctx; + char *tag = HG_HDR(payload); + const char *file, *func; + unsigned long line; + + /* + * The walker derives cell addresses from chunk bookkeeping rather than + * from a live-cell list, so a single corrupted chunk hands us an + * address that need not be mapped - and this is a DIAGNOSTIC path. A + * memory dump must never be the thing that kills the process, which is + * exactly what happened on 2026-08-09 before this check existed. + */ + if (!hg_owns_any(tag)) { + hg_corrupt(NULL, HG_C_FOREIGN_PTR); + LM_CRIT("%s: dump walker produced %p, outside every arena - " + "skipping it\n", HG_MALLOC_NAME, payload); + return; + } + + /* + * Read the DBG fields directly from the tag region, like + * fm_status_dbg reads f->file/func/line straight off the frag + * struct - NOT via hg_frag_file()/func()/line() (hg_malloc.h), + * which are nested inside "#ifdef SHM_EXTRA_STATS" (they exist only + * to serve the shm_frag_file/func/line stats ladder). This function + * is gated on DBG_MALLOC alone, so it must not depend on + * SHM_EXTRA_STATS also being on. + */ + file = *(const char **)(tag + HG_ROUNDTO); + if (!file) + return; /* stamped before any hg_malloc_dbg() call ever ran + * on this cell (e.g. still on its very first carve + * without having been freed+realloc'd) - matches + * fm_status_dbg's own "if (f->file)" guard */ + func = *(const char **)(tag + HG_ROUNDTO * 2); + line = *(unsigned long *)(tag + HG_ROUNDTO * 3); + + /* + * Defence in depth, on top of the walker sizing its set exactly. + * + * The walker infers liveness by absence from that set, so anything that + * leaves the set incomplete turns a FREE cell into an apparently live + * one - and a free cell's payload holds the free-list link, so these + * file/func would be pointers INTO THE ARENA rather than string + * literals. dbg_ht_update() would then consume them as strings. + * + * hg_owns_any() is the exact discriminator: a real __FILE__ lives in the + * binary's rodata and can never be inside an arena; a free-list link + * always is. Costs nothing on a diagnostic path, and turns what was an + * abort into a skipped line. + */ + if (hg_owns_any((void *)file) || hg_owns_any((void *)func)) { + c->skipped_notlive++; + return; + } + + if (dbg_ht_update(*c->allocd, file, func, line, hg_frag_size(payload)) < 0) + LM_ERR("unable to update the %s allocation summary\n", HG_MALLOC_NAME); +} + +/* + * f_malloc-equivalent per-allocation-site summary: walk every live cell/ + * frag, aggregate by (file,func,line) via mem_dbg_hash (mem_dbg_hash.c - + * same plain-malloc'd, allocator-independent structure fm_status_dbg + * uses), dump, free. See hg_arena_walk_live()'s comment in hg_arena.c for + * the one real accuracy caveat: exact pre-fork, best-effort post-fork + * (can't see cells idling in ANOTHER worker's private free stack) - large + * frags (hg_large_walk_live) don't share that caveat, they're always exact. + */ +void hg_status_dbg(struct hg_block *hb) +{ + mem_dbg_htable_t allocd; + struct hg_dbg_dump_ctx ctx; + struct mem_dbg_entry *it; + unsigned int i; + + LM_GEN1(memdump, "hg_status_dbg (%p):\n", hb); + if (!hb) + return; + + hg_status(hb); + + dbg_ht_init(allocd); + ctx.allocd = &allocd; + ctx.skipped_notlive = 0; + + hg_arena_walk_live(hb, hg_dbg_dump_cb, &ctx); + hg_large_walk_live(hb, hg_dbg_dump_cb, &ctx); + + LM_GEN1(memdump, " dumping summary of all alloc'ed. fragments:\n"); + LM_GEN1(memdump, "------------+---------------------------------------\n"); + LM_GEN1(memdump, "total_bytes | num_allocations x [file: func, line]\n"); + LM_GEN1(memdump, "------------+---------------------------------------\n"); + for (i = 0; i < DBG_HASH_SIZE; i++) { + for (it = allocd[i]; it; it = it->next) + LM_GEN1(memdump, " %10lu : %lu x [%s: %s, line %lu]\n", + it->size, it->no_fragments, it->file, it->func, it->line); + } + LM_GEN1(memdump, "----------------------------------------------------\n"); + if (ctx.skipped_notlive) + LM_GEN1(memdump, " %lu cell(s) skipped: header held free-list linkage, " + "so the cell was free despite not being in the free set\n", + ctx.skipped_notlive); + + dbg_ht_free(allocd); +} +#endif + +/* + * hg_cell_alloc()/hg_cell_free() (hg_arena.c, a separate translation unit) + * have a signature fixed ONCE by hg_arena.h's *original* DBG_MALLOC state - + * unlike f_malloc.c's internal helpers (fm_split_frag etc.), which live + * inside f_malloc_dyn.h itself and get recompiled fresh on each pass below, + * so their signature tracks the local #undef/#define correctly. + * hg_malloc_dyn.h's PASS 2 (after the #undef below) must therefore NOT use + * a bare "#ifdef DBG_MALLOC" to decide the hg_cell_alloc()/hg_cell_free() + * call arity - that macro is locally stale during pass 2. This sentinel + * captures the true, original state before any undef games. */ +#ifdef DBG_MALLOC +#define HG_CELL_TAKES_DBG_ARGS 1 +#else +#define HG_CELL_TAKES_DBG_ARGS 0 +#endif + +#include "hg_malloc_dyn.h" + +#if !defined INLINE_ALLOC && defined DBG_MALLOC +#undef DBG_MALLOC +#include "hg_malloc_dyn.h" +#define DBG_MALLOC +#endif + +#endif /* HG_MALLOC */ diff --git a/mem/hg_malloc.h b/mem/hg_malloc.h new file mode 100644 index 00000000000..0f86f8792f7 --- /dev/null +++ b/mem/hg_malloc.h @@ -0,0 +1,1161 @@ +/* + * hugepage-backed slab allocator + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef hg_malloc_h +#define hg_malloc_h + +#include +#include "meminfo.h" +#include "common.h" +/* for process_no, used by hg_pstat_mine() below to pick this process's + * stats slot. Included explicitly rather than relied on to arrive via + * some other header - it does on a native build, and does not under a + * cross-compiler. */ +#include "../globals.h" + +#undef HG_ROUNDTO + +#if defined(__CPU_sparc64) || defined(__CPU_sparc) + #define HG_ROUNDTO sizeof(long long) +#else + #define HG_ROUNDTO sizeof(void *) +#endif + +#define HG_NCLASSES 21 +#define HG_CELL_MAX 65536 /* largest cell; bigger allocs fail (Phase 1) */ + +/* the four-tier huge-page ladder, best first (ported from cachedb_perf) */ +enum hg_mem_tier { + HG_MEM_HUGETLB = 1, /* mmap MAP_HUGETLB */ + HG_MEM_THP_ADVISE, /* shmem THP via MADV_HUGEPAGE, huge at fault */ + HG_MEM_THP_COLLAPSE, /* shmem THP via MADV_COLLAPSE, post-fill */ + HG_MEM_4K, /* plain pages - always works */ +}; + +const char *hg_mem_tier_str(enum hg_mem_tier tier); + +/* + * Cell header, hidden before every returned pointer (like FM_FRAG(p) in + * f_malloc) - unlike cachedb_perf, HG_MALLOC hands pointers to arbitrary + * caller code via shm_malloc()/pkg_malloc(), so the class tag can NOT live + * in-band at the front of the payload the way cachedb_perf's records do. + * + * Composable layout - DBG_MALLOC and SHM_EXTRA_STATS are INDEPENDENT build + * flags (a plain non-DBG build can still have SHM_EXTRA_STATS on for + * mem-group accounting), so each contributes its own slice rather than one + * replacing the other, mirroring how f_malloc's fm_frag has both the + * DBG_MALLOC file/func/line fields AND the SHM_EXTRA_STATS + * statistic_index field as separate #ifdef'd struct members: + * + * offset 0 : class id (immutable, stamped at + * chunk-carve time) + * offset HG_ROUNDTO (DBG) : file + * offset HG_ROUNDTO*2 (DBG) : func + * offset HG_ROUNDTO*3 (DBG) : line + * offset HG_ROUNDTO+HG_CELL_HDR_DBG + * (SHM_EXTRA_STATS) : statistic_index (mem-group index) + * offset HG_CELL_HDR : payload starts here; while a cell + * is FREE, the first HG_ROUNDTO*2 bytes + * of payload double as the free-list + * link (cell_next()/cell_set_next() + * in hg_arena.c) - safe, since + * nobody reads payload of a free cell. + */ +#ifdef DBG_MALLOC +#define HG_CELL_HDR_DBG (HG_ROUNDTO * 3) /* file ptr + func ptr + line */ +#else +#define HG_CELL_HDR_DBG 0 +#endif + +#ifdef SHM_EXTRA_STATS +#define HG_CELL_HDR_STATS (HG_ROUNDTO) /* statistic_index */ +#else +#define HG_CELL_HDR_STATS 0 +#endif + +/* + * Payloads must be aligned for the widest scalar a caller may store in + * them; 8 covers uint64_t/double everywhere we build. This is NOT implied + * by HG_ROUNDTO: on 32-bit ARM HG_ROUNDTO is 4, so the raw header below would be + * 4 bytes in a plain build, and since cells always start 32-byte aligned + * EVERY payload would land at 4 mod 8 - misaligned for any 64-bit field, + * and an outright fault for the LDREXD/STREXD that gen_lock_t and the + * 64-bit atomics in shared structs compile down to. (f_malloc does not hit + * this only because its header is a struct that happens to be 8-aligned.) + * + * The padding goes at the END of the header, so every field offset below + * stays exactly where it was and only the header's total size grows. + */ +#define HG_PAYLOAD_ALIGN 8 +#define HG_CELL_HDR_RAW (HG_ROUNDTO + HG_CELL_HDR_DBG + HG_CELL_HDR_STATS) +#define HG_CELL_HDR \ + (((HG_CELL_HDR_RAW + HG_PAYLOAD_ALIGN - 1) / HG_PAYLOAD_ALIGN) \ + * HG_PAYLOAD_ALIGN) + +/* offset of the statistic_index field, valid only when SHM_EXTRA_STATS */ +#define HG_CELL_STATS_OFF (HG_ROUNDTO + HG_CELL_HDR_DBG) + +#define HG_HDR(p) ((char *)(p) - HG_CELL_HDR) +#define HG_CLASS(p) (*(unsigned char *)HG_HDR(p)) + +/* valid only when SHM_EXTRA_STATS; uniform for small cells AND large frags, + * since the tag region (and thus this offset) always sits immediately + * before payload regardless of what precedes it (fixed chunk cell vs. + * hg_lfrag boundary-tag header) */ +#ifdef SHM_EXTRA_STATS +#define HG_STATS_IDX(p) (*(unsigned long *)(HG_HDR(p) + HG_CELL_STATS_OFF)) +#endif + +/* tag-byte value marking a cell as belonging to the large-object tier + * (hg_large.c) rather than a fixed size class - valid classes are + * [0, HG_NCLASSES), so this is the first value past them, still + * distinguishable from genuine corruption (any other out-of-range byte) */ +#define HG_LARGE_MARKER HG_NCLASSES + +struct hg_large_chunk; /* opaque here, defined in hg_large.c */ +struct hg_lfrag; /* opaque here, defined in hg_large.h */ + +/* manifest sizeof(struct hg_lfrag): needed here (opaque type, can't call + * sizeof() on it) to locate a large frag's header from a payload pointer. + * hg_large.c static_asserts this matches the real struct, so any future + * field change there fails the build here instead of drifting silently. */ +#define HG_LFRAG_HDR_SIZE (4 * HG_ROUNDTO) + +/* + * A chunk is one buddy block dedicated to one cell class, and its header sits + * in the FIRST bytes OF THAT BLOCK. + * + * In-block rather than in a side array, and the arithmetic is the whole + * argument: at ~32 bytes per block, a separate array over a 5 GB arena would + * be 20 MB - twenty times every other piece of metadata combined - where + * in-block costs nothing external, shares a cache line with the block being + * touched anyway, and gives up exactly one cell. + * + * The enabler was already in the tree before any of v2: hg_cell_free() reads + * the class from the CELL's own header byte rather than from the chunk, so a + * page can host blocks of different classes with no fast-path change. + */ +/* + * Fullness grades. Eight is the design's figure: enough that "fullest + * partial" is meaningfully sorted, few enough that a block changes grade + * rarely rather than on every cell, and that the scan for a source block is + * eight pointer tests. + */ +#define HG_GRADES 8 +#define HG_GRADE_NONE 0xffffffffu /* no free cells, on no list */ +#define HG_GRADE_DRAINED 0xfffffffeu /* every cell free, on drained[] */ + +/* a block whose cells are ALL in the global pool - the GC's work queue */ +#define HG_CHUNK_DRAINED (1u << 0) +/* selected by this GC pass; its cells are being unlinked right now */ +#define HG_CHUNK_RECLAIMING (1u << 1) + +struct hg_chunk { + /* + * Global registry. Doubly linked, which it did not need to be while + * chunks were immortal: hg_slab_recycled() and the DBG dump walker both + * traverse it, so a reclaimed block MUST come out of it - otherwise the + * first keeps counting capacity that no longer exists and the second + * reads a block the buddy has already handed to another class. + */ + struct hg_chunk *next; + struct hg_chunk *prev; + unsigned int cls; /* immutable */ + unsigned int cell_size; /* total slot size, header included */ + unsigned int cells; + + /* + * Cells of this block currently parked in the GLOBAL pool, i.e. + * definitively free and reachable by anyone. Maintained only at + * cache/block transitions (gpool_push/gpool_pop), which are 1-3% of + * operations and already under hb->lock - a free that lands in a + * thread's private LIFO does not touch this, by design. + * + * A cell sitting in some thread's private cache therefore still counts + * as live. That is deliberately conservative: it can only delay a + * reclaim, never cause a premature one, and flushing those caches is + * exactly what the idle sweep (task #59) is for. + * + * in_gpool == cells means every cell of this block is free and globally + * visible, which is the condition the GC in task #57 acts on. + */ + unsigned int in_gpool; + unsigned int flags; /* HG_CHUNK_DRAINED / _RECLAIMING */ + unsigned int order; /* buddy order, so the GC can hand it back */ + + /* + * This block's OWN free cells, and which fullness list it currently sits + * on. The shared pool used to be one mixed list per class, so a 32-cell + * refill skimmed cells from whichever blocks happened to be at its head + * and topped every block up a little - measured, that left blocks + * stalled at 88% drained and the GC with nothing to collect. + * + * Per block, a refill can instead take all 32 from ONE block, and the + * fullness grading makes it the FULLEST partial - so that block is used + * up while emptier ones are left alone to reach zero. + * + * This costs the fast path nothing: hg_cell_free() still pushes to the + * thread-private TLS LIFO with no lock, and only the donation/refill + * path - already under hb->lock, 1-3% of operations - touches this. + */ + void *free_head; + unsigned int grade; /* HG_GRADE_NONE / _DRAINED, or 0..HG_GRADES-1 */ + + /* + * The list this block is currently on. Today that is only the per-class + * drained queue; task #58 generalises it into fullness buckets, of + * which "drained" is simply the empty one - which is why the design + * calls the empty bucket the GC's work queue. + */ + struct hg_chunk *fnext; + struct hg_chunk *fprev; +} __attribute__ ((aligned (64))); + +/* + * v2 buddy geometry - mem/README.hg_arena_v2, "Address to block". + * + * The leaf is the minimum buddy order. 8 KB is a deliberate floor: the leaf + * array is one byte per leaf per page, so dropping to 1 KB would multiply that + * array by eight for no measured gain. Do not lower it without evidence. + * + * Leaves per page is NOT a constant, because the huge page size is probed - + * 256 leaves on a 2M page, 65536 on a 512M arm64 page - so it is derived from + * hb->hps_shift rather than baked in. + */ +#define HG_LEAF_SHIFT 13 +#define HG_LEAF_SIZE (1UL << HG_LEAF_SHIFT) + +/* + * Ceiling on the number of buddy orders, for the fixed free-list array in + * struct hg_block. The REAL count is derived per arena as + * hps_shift - HG_LEAF_SHIFT: 8 on a 2 MB page with 8 KB leaves, 16 on a + * 512 MB arm64 page. 24 leaves room for a 128 GB page that does not exist yet + * and costs 25 pointers in one struct. + */ +#define HG_MAX_ORDERS 24 + +struct hg_page; /* hg_buddy.h */ +struct hg_free_blk; /* hg_buddy.h */ + +/* the accessors that turn an address into a page and a leaf live just after + * struct hg_block below - they dereference it, so they cannot precede it */ + +struct hg_region { + struct hg_region *next; + unsigned long size; +}; + +/* per-process private allocation state for ONE hg_block instance. Several + * hg_block instances can be live in the same process at once (shm, shm_dbg, + * pkg all selecting HG_MALLOC simultaneously) so this is looked up per-block, + * not a single global - see hg_get_palloc() in hg_arena.c */ +struct hg_palloc { + struct hg_block *owner; /* which block this state belongs to */ + struct { + char *bump; /* next unused cell in own chunk */ + unsigned int left; + void *free_head; /* private LIFO free stack */ + unsigned int nfree; + } cls[HG_NCLASSES]; +}; + +/* + * Per-process counters for the two stats the lock-free fast path has to + * touch on every single allocation and free. + * + * Keeping them as plain fields in hg_block would make every worker do an + * unsynchronized read-modify-write on the same shared words - a data race + * on every architecture (x86 TSO does not make "x += y" atomic either; + * a weakly-ordered machine just loses more updates), whose symptom is + * silently under-reported memory in /info. Making them atomic instead + * would be correct but would put a contended shared cache line back on + * the fast path - exactly what the per-process free stacks exist to + * avoid. So each process gets its own cache-line-isolated slot and + * readers sum the slots. + * + * Both fields are SIGNED on purpose: a cell allocated by one process can + * be freed by another (that is what the shared pool is for), so an + * individual slot legitimately goes negative. Only the sum is meaningful. + */ +#define HG_STAT_SLOTS 256 +#define HG_STAT_LINE 64 + +struct hg_pstat { + long used; /* payload bytes handed out by this process */ + long fragments; /* live cells handed out by this process */ + /* Cell-slot bytes (header + payload + size-class round-up) currently + * handed out by this process. "used" alone cannot tell how much ARENA + * a process is holding, because a 100-byte request occupies a whole + * 128-byte slot; hg_slab_recycled() needs the slot figure to work out + * how much carved capacity is sitting idle. Lives on the same + * already-private cache line as the two counters above, so maintaining + * it costs no extra cache traffic on the fast path. */ + long cell_live; + char _pad[HG_STAT_LINE - 3 * sizeof(long)]; +} __attribute__ ((aligned (HG_STAT_LINE))); + +/* + * What kind of corruption a check caught. Grouped by what an operator would do + * about it rather than by which line found it: several sites detect the same + * defect from different angles, and splitting them would make a recurrence + * look like several unrelated rare events instead of one repeated one. + */ +enum hg_corrupt_kind { + HG_C_CLASS_MISMATCH = 0, /* cell resolves to a block of another class */ + HG_C_DOUBLE_FREE, /* block already fully free, or buddy re-freed */ + HG_C_NFREE_UNDERFLOW, /* free list shorter than nfree claims - THIS */ + /* is the __thread palloc_slots signature */ + HG_C_BAD_CLASS, /* cell header carries an impossible class id */ + HG_C_FOREIGN_PTR, /* pointer belongs to no live arena */ + HG_C_BUDDY_BAD_FREE, /* outside the grid, misaligned, wrong order */ + HG_C_INTERNAL, /* allocator API used in a way that would */ + /* double-allocate - a bug here, not in a peer */ + HG_CORRUPT_KINDS +}; + +/* for the two checks that fire where no arena pointer exists */ +extern unsigned long hg_corrupt_noarena[HG_CORRUPT_KINDS]; + +struct hg_block { + char *name; /* purpose of this memory block */ + + gen_lock_t lock; /* slow paths only: gpool + chunk carve */ + + struct hg_chunk *chunks; + unsigned int nchunks; + /* upper bound on one chunk, derived from the arena size at init - + * see chunk_size_for() in hg_arena.c */ + unsigned int chunk_max; + /* Per-THREAD private free-cache bound, in CELLS, one entry per class, + * derived from the arena size at init - see private_caps_init() in + * hg_arena.c. A flat cell count cannot bound anything, because the + * same count means 16 KB in class 64 and 16 MB in class 65536; these + * are the byte budget expressed per class. priv_max may legitimately + * be 0, which means "never cache this class privately" - the class is + * large enough that one cell already exceeds a thread's whole share, + * and such allocations are rare enough that the shared pool is the + * right home for them. */ + unsigned int priv_max[HG_NCLASSES]; + unsigned int priv_donate[HG_NCLASSES]; + struct hg_region *regions; + /* + * Shared free cells, per class, held as BLOCKS graded by fullness + * rather than as one mixed cell list. bucket[c][0] holds the blocks + * with the fewest free cells, so scanning from 0 up finds the fullest + * partial in O(HG_GRADES) - the "concentration" the design requires. + * A block with no free cells is on no list; one with every cell free is + * on drained[] instead, which is the GC's queue. + */ + struct hg_chunk *bucket[HG_NCLASSES][HG_GRADES]; + unsigned int gpool_n[HG_NCLASSES]; /* free cells of the class, total */ + unsigned long lo, hi; /* extent watermarks */ + + /* large-object tier (hg_large.c): list of independently-carved + * chunks, each an f_malloc-style boundary-tag heap, sharing ONE free + * list across all of them; hb->lock guards all of it (slow path + * already, no separate lock needed) */ + struct hg_large_chunk *large_chunks; + struct hg_lfrag *large_free; + /* + * Backing the large tier holds from the buddy grid, and how much of it + * is handed out right now. Both are needed because the two answer + * different questions and only the first one is what the grid lost: + * a chunk is taken whole from the buddy, then sub-allocated, so + * charging only the live fragments (which is what this tier used to do) + * left real_used under-reporting the arena's true footprint by the + * chunks' unused slack - and free_to_carve over-reporting what was + * left, monotonically, which is exactly the v1 pathology v2 exists to + * remove. large_backing is charged to real_used at chunk acquisition; + * large_recycled() = large_backing - large_live is the large tier's + * analogue of hg_slab_recycled() and is subtracted back out to get the + * live figure. + */ + unsigned long large_backing; + unsigned long large_live; + /* chunk churn: a chunk goes back to the buddy the moment its last + * fragment is freed, so carved climbing far faster than returned is + * the signature of alloc/free thrash in this tier */ + unsigned long large_chunks_carved; + unsigned long large_chunks_returned; + + /* Bytes carved from the reservation into chunks/regions, i.e. the + * arena's own footprint. Only ever changed while holding hb->lock + * (carve_chunk, hg_region_alloc, the large tier), never on the + * lock-free fast path, so a plain field is safe here. */ + unsigned long real_used; + unsigned long max_real_used; + /* High-water mark of the LIVE figure that hg_get_real_used() reports, as + * opposed to max_real_used above, which is the peak CARVE. They are + * different quantities: carve only ever grows (chunks are never + * un-carved), so reporting it as max_used made max_used drift away from + * real_used forever instead of meaning "the peak real_used reached" - + * unlike every other allocator. Sampled, not exact: refreshed whenever + * the stats are read, so a spike between two reads can be missed. The + * unsynchronized max update is a benign race - a lost update can only + * under-report, never over-report. */ + unsigned long max_live_used; + + /* the fast-path counters - see struct hg_pstat above. Summed by + * hg_used()/hg_fragments(); never read directly. */ + struct hg_pstat pstat[HG_STAT_SLOTS]; + unsigned long size; /* total arena size */ + + /* the huge-page reservation this block owns: a pre-fork (or per-process, + * for PKG), never-unmapped, 2M-aligned MAP_SHARED (or private, for PKG) + * region. Chunks bump from it lock-free (atomic hoff) */ + char *hbase; + unsigned long hsize; + volatile unsigned long hoff; + enum hg_mem_tier tier; + unsigned long locked_mb; + /* MAP_SHARED vs MAP_PRIVATE, as passed to hg_malloc_init(). Stored + * because growth policy depends on it: a pkg delta is per PROCESS, + * so its host-RAM cost is delta x nproc, and the ceiling check must + * know which multiplication to apply. */ + int shared; + + /* + * v3 elastic arena. + * + * hsize is what is COMMITTED - pre-faulted, pinned, and published to + * the buddy. hcap is what is RESERVED in virtual address space. The + * WHOLE cap is mapped once, before fork; growth only commits more of + * what is already mapped. + * + * That split is forced, not stylistic. mmap() and mprotect() edit ONE + * process's page tables, and the shm arena is shared by ~30 workers + * that forked before any growth happens. A delta mapped into a + * PROT_NONE reservation after fork is invisible to every one of them: + * measured, the grower reads its new page fine and a forked worker + * SIGSEGVs on the same address. Mapping the cap up front gives every + * worker one VMA over one shmem object, so a page the grower commits + * simply faults in wherever it is next touched. + * + * hcap == hsize is a fixed arena - exactly v2's behaviour, and the + * default until an admin asks for more. + */ + unsigned long hcap; /* VA reserved, >= hsize */ + unsigned long hsize_min; /* hsize at init - shrink's floor. + * The admin asked for -m/-M of + * memory; growth above it is + * elastic, the base is not. */ + unsigned long grow_granule; /* bytes per grow step, hps multiple */ + unsigned long grows; /* successful commits */ + unsigned long grow_bytes; /* their total */ + unsigned long grow_refused; /* refusals (cap or resource) */ + unsigned long shrinks; /* successful releases */ + unsigned long shrink_bytes; /* their total */ + /* + * v3 step 4: the attached auto-scaling POLICY - numbers copied out of + * a config auto_scaling_profile at attach time, never a pointer (the + * profile struct lives in process-local memory; this block may be + * shared). active==0 means no profile: growth still works up to hcap + * on exhaustion, shrink keeps its conservative built-in gate - the + * step-1..3 behaviour, unchanged. + * + * The profile's worker-count fields map onto bytes: "scale up to N" + * is N MB, hps-rounded, and becomes the ADMIN CEILING within the + * -m INIT:CAP reservation (it can never raise hcap - the reservation + * happened before the config was even parsed, which is the whole + * reason the cap lives on the command line). "down to M" is the + * shrink floor and MAY sit below the initial size: with a profile + * attached, the profile is what the admin asked for, -m is just the + * starting point. + */ + struct { + unsigned int active; + unsigned long up_bytes; /* admin ceiling, <= hcap */ + unsigned long down_bytes; /* shrink floor */ + unsigned int up_pct; /* grow when usage >= this... */ + unsigned int up_need; /* ...for this many ticks... */ + unsigned int up_window; /* ...out of this window */ + unsigned int down_pct; /* shrink when usage <= this... */ + unsigned int down_cycles; /* ...for this many consecutive */ + unsigned short cooldown; /* post-grow shrink hold-off */ + } pol; + unsigned int pol_up_hits; /* ticks over up_pct in window */ + unsigned int pol_up_ticks; /* window position */ + unsigned int pol_cooldown; /* ticks left before shrink counts */ + unsigned int pol_dry_said; /* one advise line per episode */ + /* consecutive quiet sweep ticks - the down-slow gate. Reset by any + * grow and by any tick that fails the abundance test, so a shrink + * needs a full uninterrupted quiet window. */ + unsigned int shrink_quiet; + /* set once when hg_mem_release() fails structurally (e.g. a kernel + * without hugetlb hole punch): shrink is disabled for this arena's + * lifetime rather than re-attempted and re-logged every window */ + unsigned int shrink_unsupported; + /* + * One line per refusal EPISODE, not per refusal: a full arena refuses + * on every subsequent allocation (measured: 239k NOTICEs in 4s on the + * first at-cap soak), and the counter above already carries the + * magnitude. Set when a refusal is logged, cleared by the next + * successful grow. + */ + unsigned int grow_refuse_said; + /* + * The alertable grow-blocked state, RESOURCE refusals only - an admin + * cap doing its job is policy, not an incident, and never latches. + * + * Latching is two-step, modelled on below_floor: the first resource + * refusal only records the GC-pass count it must outlive + * (grow_blocked_mark = gc_passes + 1); the latch arms when a refusal + * recurs at gc_passes >= that mark, i.e. a full GC pass ran in + * between and the arena STILL cannot grow - so a transient spike that + * one reclaim pass absorbs never alerts. Cleared by a successful + * grow (the resource came back) or by free space recovering above + * the reserve floor (the demand went away) - the design's "clear + * when demand falls below a lower mark", reusing the floor's own + * hysteresis threshold rather than inventing a second one. + * + * grow_blocked is the gauge the statistics export; grow_event_due + * hands the E_CORE_SHM_GROW_BLOCKED raise to the sweep timer, which + * runs with no arena lock held - evi_raise_event() allocates shm, + * and raising it here, under hb->lock, inside the allocator that + * just refused, would be re-entry into a full arena at best. + */ + unsigned int grow_blocked; + unsigned long grow_blocked_mark; + /* + * grow_refused at arming time. The gc_passes route above assumes GC + * RUNS; on the state that matters most - a full arena where nothing + * is reclaimable - gc_passes sits at zero forever and the latch + * would never arm (measured: 5M refusals, gc_passes 0, no latch). + * So the sweep timer is the fallback promoter: if a full sweep + * interval passes with the episode still armed and refusals still + * accumulating, it latches from there. A spike that ends before the + * next sweep still never alerts. + */ + unsigned long grow_blocked_refuse0; + unsigned int grow_event_due; + /* + * Bytes of the arena per ACHIEVED backing tier, indexed by + * enum hg_mem_tier (slot 0 unused; the enum starts at 1). hb->tier + * alone cannot describe a grown arena: every THP delta is a fresh + * negotiation with the kernel and may land on 4K next to an arena + * that got 2M at init. Page backing is an outcome per range, never + * an attribute of the arena - report it as such. + */ + unsigned long tier_bytes[HG_MEM_4K + 1]; + + /* + * v2 buddy substrate - see mem/README.hg_arena_v2. + * + * The whole design turns "which block owns this address" into two shifts + * and a mask, which needs a known-aligned origin. hbase is NOT reliably + * that origin: all three Linux reserve paths align it (MAP_HUGETLB is + * aligned by the kernel, the THP tiers align explicitly at + * hg_malloc.c:301), but the non-Linux fallback takes a plain + * mmap(NULL, ...) and gets only page alignment. So page 0 starts at + * pbase, hbase rounded up to a huge page - equal to hbase everywhere it + * matters, and correct where it is not. + * + * hps is PROBED, not assumed: it is 2M on x86_64 and on arm64 with 4K + * base pages, but 32M with 16K pages and 512M with 64K. A hardcoded + * ">> 21" would mis-address every block on those machines, which is the + * same trap hg_hps() already exists to avoid for the mapping itself. + */ + unsigned long hps; /* huge page size, probed at reserve time */ + unsigned int hps_shift; /* log2(hps), so page-of is a shift */ + char *pbase; /* page 0 - hbase rounded up to hps */ + unsigned long npages; /* whole pages from pbase to the COMMITTED end + * (hbase+hsize); grows when the arena does */ + unsigned long npages_cap; /* whole pages to the reservation end + * (hbase+hcap) - the grid's true extent. + * Descriptors exist for all of these from + * init, so growth publishes pages instead of + * relocating metadata */ + + /* + * Buddy state (hg_buddy.c). Every field here is written only while + * holding hb->lock - the buddy is entirely slow path, reached from + * carve_chunk() and the large tier, never from the cell fast path. + */ + struct hg_page *pages; /* npages descriptors */ + struct hg_free_blk *bfree[HG_MAX_ORDERS + 1]; /* free list per order */ + unsigned long nfree[HG_MAX_ORDERS + 1]; /* its length, per order */ + unsigned long buddy_free_leaves; /* free leaves, whole arena */ + unsigned int buddy_top; /* whole-page order, cached */ + unsigned int buddy_ready; /* 0 until hg_buddy_init() */ + + /* + * GC work queue: blocks whose every cell is in the global pool, per + * class. Reclaiming one means unlinking its cells from that class's + * free list, which is a walk - so drained blocks accumulate here and a + * single walk serves all of them, rather than one walk per block. + */ + struct hg_chunk *drained[HG_NCLASSES]; + unsigned int ndrained[HG_NCLASSES]; + unsigned long gc_blocks_returned; /* lifetime, for stats */ + unsigned long gc_passes; + unsigned long cache_flushes; /* sweeps run */ + unsigned long cells_flushed; /* cells recovered from TLS */ + /* + * Corruption counters, one per kind. + * + * Every consistency check in this allocator used to emit LM_CRIT and + * increment nothing, so a log grep was the ONLY detector - and on the + * production gateways the log sink is a file that journalctl does not + * see, while OpenSIPS also logs unrelated DNS failures at CRITICAL. A + * recurrence of the tcp payload use-after-free would move no number at + * all. These make it numeric, so it can be alerted on and graphed. + * + * They count DETECTIONS, not repairs: every site that bumps one has + * already decided to refuse the operation or leak the cell. A non-zero + * value means memory was corrupted and the allocator noticed - it is + * never routine. + */ + unsigned long corrupt[HG_CORRUPT_KINDS]; + unsigned long buddy_splits; /* blocks split down an order */ + /* + * RUNTIME merges only. hg_buddy_init() publishes the one page that the + * block header and buddy metadata straddle leaf by leaf, through the + * ordinary free path, so the tree is built by the normal rules - and + * every one of those coalesces used to land in this counter. That gave + * it an arbitrary startup offset with no relation to fragmentation: + * an idle 8 MB pkg arena reads 7 splits against 244 merges purely from + * init. The init total is snapshotted into buddy_merges_init and this + * counter is rebased to 0, so splits and merges finally share a zero + * point and their difference means something. + */ + unsigned long buddy_merges; /* blocks merged with a buddy */ + unsigned long buddy_merges_init; /* coalesces done building the tree */ + unsigned long blocks_carved; /* class blocks cut, lifetime */ + /* set while a flush walks a cache chain: pushing a cell can reclaim its + * block, and the next cell on the chain may live in that same block */ + unsigned int gc_deferred; + /* + * Reserve floor: free leaves below which the arena is treated as under + * pressure. Set once at init as a fraction of the grid. Not consumable + * by an expanding class in the sense that crossing it triggers a sweep + * BEFORE the failure, which is the only warning an operator can act on. + */ + unsigned long reserve_floor; + unsigned int below_floor; /* hysteresis: already reported */ + unsigned long floor_crossings; + + unsigned char size2class[(HG_CELL_MAX / HG_ROUNDTO) + 1]; +} __attribute__ ((aligned (HG_ROUNDTO))); + +/* + * Address -> page/leaf, the arithmetic the whole v2 buddy layer rests on. + * See the HG_LEAF_SHIFT block above and mem/README.hg_arena_v2. + * pages_init() in hg_arena.c verifies these against real addresses at every + * arena init, and refuses to start the arena if they do not round-trip. + */ + +/* how many leaves tile one huge page */ +static inline unsigned long hg_leaves_per_page(const struct hg_block *hb) +{ + return 1UL << (hb->hps_shift - HG_LEAF_SHIFT); +} + +/* Is @p inside the page-addressable region? Everything before pbase (the + * block header, and on a non-Linux fallback the unaligned head) is arena + * memory but not buddy memory, so it must answer NO. */ +static inline int hg_in_pages(const struct hg_block *hb, const void *p) +{ + return (const char *)p >= hb->pbase && + (const char *)p < hb->pbase + (hb->npages << hb->hps_shift); +} + +/* page index of @p; only meaningful when hg_in_pages() */ +static inline unsigned long hg_page_of(const struct hg_block *hb, const void *p) +{ + return (unsigned long)((const char *)p - hb->pbase) >> hb->hps_shift; +} + +/* first byte of the page holding @p */ +static inline char *hg_page_base(const struct hg_block *hb, const void *p) +{ + return hb->pbase + (hg_page_of(hb, p) << hb->hps_shift); +} + +/* leaf index of @p WITHIN its own page */ +static inline unsigned long hg_leaf_of(const struct hg_block *hb, const void *p) +{ + unsigned long off = (unsigned long)((const char *)p - hb->pbase); + + return (off & ((1UL << hb->hps_shift) - 1)) >> HG_LEAF_SHIFT; +} + +/* + * Reserves its own huge-page-backed (or gracefully degraded) region of + * @size bytes and lays out the block control structure at its start. + * + * Unlike fm_malloc_init(), this does NOT take a pre-reserved address: every + * other allocator receives memory that shm_getmem() already mmap'd (a plain, + * non-huge anonymous mapping), but HG_MALLOC always needs to run its own + * hugepage tier ladder to get the mapping in the first place, so it owns the + * whole reservation step itself (see hg_malloc.c). Returns NULL on total + * failure (caller logs and aborts startup, per the "no silent fallback to a + * different allocator" design decision). + */ +/* @shared: 1 for shm/shm_dbg (MAP_SHARED - one arena for every worker), + * 0 for pkg (MAP_PRIVATE - each forked worker gets its own copy-on-write + * arena, lock and free pools). See hg_mem_reserve() in hg_malloc.c for why + * getting this wrong for pkg is a correctness AND a performance bug. + * + * @flags: HG_INIT_INHERITED marks the ONE arena that outlives fork() as a + * copy-on-write inheritance: the pre-fork (attendant) pkg arena. Every + * child holds a private COW view of it for life - it reads module state + * the parent parsed into it, and any write it makes there is a COW fault. + * Such an arena must never be hugetlb-backed: a COW fault inside a + * hugetlb VMA can only be satisfied by a huge page (no 4K fallback), and + * a forked child holds no reservation on its parent's mapping, so with + * the pool momentarily empty the fault is a SIGBUS - silent, at fork + * time (measured: it is exactly how the last no-script child, TCP main, + * died at startup on a short pool). THP has the fallback: a COW fault + * splits the huge PMD and copies one 4K page. Per-child arenas are never + * inherited (children do not fork) and keep the full ladder. */ +#define HG_INIT_INHERITED (1U << 0) + +struct hg_block *hg_malloc_init(unsigned long size, char *name, int shared, + const char *proc_desc, unsigned int flags); +void hg_malloc_destroy(struct hg_block *hb); + +/* + * The v3 admin caps live in globals.h/globals.c (set by -m INIT:CAP / + * -M INIT:CAP before any arena exists; 0 = fixed, exactly v2). The pkg + * cap is PER PROCESS: every worker grows its own private arena, so the + * host-RAM exposure is cap x nproc - hg_grow_ram_refused() applies that + * multiplication. + */ + +/* + * Commit @delta more bytes at committed-end offset @off of the reservation + * (both hps-rounded), populating and pinning them, and VERIFYING what + * backing the kernel actually provided. Returns the achieved tier of the + * delta (>= 1) or -1 with everything rolled back - a refusal, not a + * degradation, so a worker never SIGBUSes on memory the allocator + * half-committed. Defined in hg_malloc.c because the tier ladder and its + * verification probes live there; called by hg_buddy_grow() under + * hb->lock. + */ +int hg_mem_commit(struct hg_block *hb, unsigned long off, unsigned long delta); + +/* + * The host-RAM limb of the growth ceiling: would committing @delta more + * bytes leave the host with less than the configured floor of available + * memory? Returns nonzero to REFUSE. Tier 1 always passes - a hugetlb + * mapping reserved its whole cap from the pool at map time (measured), so + * its commits consume no new host RAM. For pkg arenas the delta is + * multiplied by the process count first: every worker grows its own + * private arena under the same workload, so the single-arena delta + * understates the real cost ~30x on a gateway. Defined in hg_malloc.c + * (it owns /proc/meminfo parsing); called by hg_buddy_grow() under + * hb->lock. + */ +int hg_grow_ram_refused(struct hg_block *hb, unsigned long delta); + +/* + * Release the backing of [hbase+off, +len) - the shrink primitive, chosen + * and verified by measurement on the fleet's oldest kernel (5.4): + * + * shared (shm): madvise(MADV_REMOVE) - punches the shmem OBJECT, so + * every mapper is affected; measured to free pages even + * while another process holds them VM_LOCKED, and the + * range recommits cleanly afterwards. The one mechanism + * that is NOT correct here is mmap(PROT_NONE|MAP_FIXED): + * it rebinds only the caller's mapping and was measured + * leaving other workers reading the old bytes. + * shared tier 1: same call; hugetlb hole punch works on 5.4 and the + * pages return to HugePages_Free (measured), which is + * where a static pool's shrink SHOULD put them. + * private (pkg): madvise(MADV_DONTNEED) - per-process arena, no + * cross-process question; next touch refaults zero. + * + * munlock first: it releases only THIS process's VM_LOCKED accounting - + * a range mlocked by the worker that grew it keeps its stale VmLck there + * until exit, which is cosmetic; the punch frees the memory regardless. + * Returns 0, or -1 with shrink_unsupported latched (nothing to retry). + */ +int hg_mem_release(struct hg_block *hb, unsigned long off, unsigned long len); + +/* + * Resolve and validate the configured auto-scaling profiles, once, after + * the config is parsed (called from init_shm_post_yyparse()). Attaches the + * policy to the LIVE shm arena and stashes the pkg policy for the arenas + * pt.c creates per child after fork - the pre-fork parent pkg arena + * predates the config and stays fixed, which costs nothing (the attendant + * barely allocates). Fails LOUDLY on a profile that names nothing, exceeds + * the -m/-M reservation, or is attached to an arena with no growth room: + * a policy that cannot act is a misconfiguration, not a default. + */ +int hg_autoscale_post_cfg(void); + +/* re-sync per-process state after fork(): see hg_arena.c for why the + * inherited private free-stack/bump state must be discarded, not kept or + * donated (ported from cachedb_perf's pcache_arena_child_init reasoning) */ +void hg_malloc_child_init(struct hg_block *hb); + +/* sizes the DBG_MALLOC allocation-history pool (shm_hist / struct_hist), + * same purpose and shape as fm_get_dbg_pool_size() - a HG_CELL_HDR-based + * estimate substituted for f_malloc's FRAG_OVERHEAD. Best-effort: under- or + * over-estimating just wastes a bit of the -m reservation or fails a chunk + * carve loudly (logged, not corrupting) - no correctness risk either way, + * since chunks are carved dynamically rather than packed into one fixed + * region the way f_malloc's frag allocator is. */ +unsigned long hg_get_dbg_pool_size(unsigned int hist_size); + +#ifdef DBG_MALLOC +void *hg_malloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line); +void hg_free(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line); +void *hg_realloc(struct hg_block *hb, void *p, unsigned long size, + const char *file, const char *func, unsigned int line); +#ifndef INLINE_ALLOC +void *hg_malloc_dbg(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line); +void hg_free_dbg(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line); +void *hg_realloc_dbg(struct hg_block *hb, void *p, unsigned long size, + const char *file, const char *func, unsigned int line); +#endif +#else +void *hg_malloc(struct hg_block *hb, unsigned long size); +void hg_free(struct hg_block *hb, void *p); +void *hg_realloc(struct hg_block *hb, void *p, unsigned long size); +#endif + +void hg_status(struct hg_block *hb); +#if !defined INLINE_ALLOC && defined DBG_MALLOC +void hg_status_dbg(struct hg_block *hb); +#endif +void hg_info(struct hg_block *hb, struct mem_info *info); + +/* defined in hg_arena.c; table lookup, no lock needed (immutable) */ +unsigned int hg_cell_total_size(unsigned char cls); + +/* defined in hg_large.c; @frag is HG_HDR(p) - HG_LFRAG_HDR (the struct + * hg_lfrag* at the very start of the block). Declared here taking an + * opaque pointer rather than struct hg_lfrag* to avoid hg_malloc.h needing + * to depend on hg_large.h, which itself includes hg_malloc.h. */ +unsigned long hg_large_frag_size_at(const void *frag); + +/* + * Arena ownership tests. + * + * Everything hg hands out lives inside some block's hb->hbase reservation, + * whose bounds are fixed at init. A pointer from outside it - a foreign + * arena, a stale pointer, or a corrupted one - has a "class byte" at + * HG_HDR(p) that is either garbage or, worse, unmapped, so simply reading it + * faults. That is not hypothetical: it took a process down (staging RGS, + * 2026-08-09, si_addr == p-32 inside hg_frag_size()). + * + * Two flavours, because the callers differ: + * + * hg_owns() - for code that already knows which block it is working on. + * Exact, one range, no loop. + * + * hg_owns_any() - for code that does NOT. hg_frag_size() is the reason this + * exists: it is installed into the shared + * "unsigned long (*shm_frag_size)(void *)" function pointer + * next to fm_/qm_/hp_/parallel_frag_size(), so its signature + * belongs to an interface we do not own and cannot grow an + * hb parameter. It walks a registry of live arenas instead. + * + * This IS on the free fast path, contrary to what this + * comment used to claim: _shm_free() calls shm_frag_size() + * unconditionally on every free (mem/shm_mem.h), so every + * shm_free walks the registry. Measured anyway, three + * alternating pairs at 800 cps on the bench harness: + * 2.747% allocator self-time without the checks, 2.807% + * with, against a within-arm spread of ~0.2pp. The cost is + * below the noise floor, so the registry stays a plain + * linear walk - a cache here would be unmeasurable + * complexity. Revisit only if HG_ARENA_REG_MAX grows. + * + * Both are advisory: they turn "dereference and die" into "decline and carry + * on". They do not make a bad pointer good. + */ +static inline int hg_owns(struct hg_block *hb, void *cell_start) +{ + return (char *)cell_start >= hb->hbase && + (char *)cell_start < hb->hbase + hb->hsize; +} + +#define HG_ARENA_REG_MAX 8 +struct hg_arena_range { + char *base; + unsigned long size; + struct hg_block *hb; +}; +/* defined in hg_malloc.c; maintained by hg_malloc_init/destroy. Process-local + * on purpose - a forked child inherits the parent's entries (its shm mapping + * really is the same memory) and adds its own private pkg arena on top. */ +extern struct hg_arena_range hg_arena_reg[HG_ARENA_REG_MAX]; + +/* count of frees redirected to their true owner; reported by hg_stats */ +extern unsigned long hg_xarena_frees; + +/* + * Which arena does this pointer belong to, if any? + * + * A child does NOT only ever free pointers from its current arena, and it is + * not a bug when it does not. Every other allocator lets a child inherit the + * parent's pkg arena COW, so freeing something the parent allocated pre-fork + * is ordinary, supported behaviour - cachedb_redis and six sibling modules do + * exactly that in child_init(), releasing the URL list mod_init() built. + * HG_MALLOC hands each child a fresh arena instead (see pt.c), which broke + * that assumption: those frees arrive addressed to an arena that never issued + * them. + * + * So resolve the owner rather than judging by the caller's block. A pointer + * that belongs to some other live arena is redirected there and freed + * properly; only a pointer that belongs to NO arena is a real defect. + */ +static inline struct hg_block *hg_owner(const void *p) +{ + int i; + + for (i = 0; i < HG_ARENA_REG_MAX; i++) { + if (!hg_arena_reg[i].base) + continue; + if ((const char *)p >= hg_arena_reg[i].base && + (const char *)p < hg_arena_reg[i].base + hg_arena_reg[i].size) + return hg_arena_reg[i].hb; + } + return NULL; +} + +static inline int hg_owns_any(const void *p) +{ + return hg_owner(p) != NULL; +} + +static inline unsigned long hg_frag_size(void *p) +{ + unsigned char c; + + if (!p) + return 0; + + /* the header may not be mapped at all - check before reading it. + * Returning 0 matches the "unknown size" answer this function already + * gives for an out-of-range class. */ + if (!hg_owns_any(HG_HDR(p))) + return 0; + + c = HG_CLASS(p); + if (c == HG_LARGE_MARKER) + return hg_large_frag_size_at(HG_HDR(p) - HG_LFRAG_HDR_SIZE); + if (c >= HG_NCLASSES) + return 0; + + return hg_cell_total_size(c); +} + +#define HG_FRAG_OVERHEAD (HG_CELL_HDR) + +#ifdef SHM_EXTRA_STATS +void hg_stats_core_init(struct hg_block *hb, int core_index); +unsigned long hg_stats_get_index(void *ptr); +void hg_stats_set_index(void *ptr, unsigned long idx); + +#ifdef DBG_MALLOC +static inline const char *hg_frag_file(void *p) +{ + if (!hg_owns_any(HG_HDR(p))) + return NULL; + return *(const char **)(HG_HDR(p) + HG_ROUNDTO); +} +static inline const char *hg_frag_func(void *p) +{ + if (!hg_owns_any(HG_HDR(p))) + return NULL; + return *(const char **)(HG_HDR(p) + HG_ROUNDTO * 2); +} +static inline unsigned long hg_frag_line(void *p) +{ + if (!hg_owns_any(HG_HDR(p))) + return 0; + return *(unsigned long *)(HG_HDR(p) + HG_ROUNDTO * 3); +} +#else +static inline const char *hg_frag_file(void *p) { return NULL; } +static inline const char *hg_frag_func(void *p) { return NULL; } +static inline unsigned long hg_frag_line(void *p) { return 0; } +#endif +#endif + +/* + * Fast-path stats helpers. + * + * hg_pstat_mine() picks this process's slot. process_no is -1 in the + * attendant and 0 in the main process before fork, so it is biased by one + * and wrapped: two processes sharing a slot would only reintroduce the + * lost-update race for those two, never corrupt anything, and with + * HG_STAT_SLOTS slots that needs a genuinely enormous process table. + */ +static inline struct hg_pstat *hg_pstat_mine(struct hg_block *hb) +{ + return &hb->pstat[((unsigned int)(process_no + 1)) % HG_STAT_SLOTS]; +} + +/* summed on read; clamped at 0 because individual slots go negative when + * one process frees another's cells and a torn sum could otherwise + * underflow an unsigned return */ +static inline unsigned long hg_used(struct hg_block *hb) +{ + long total = 0; + int i; + + for (i = 0; i < HG_STAT_SLOTS; i++) + total += hb->pstat[i].used; + return total < 0 ? 0 : (unsigned long)total; +} + +static inline unsigned long hg_fragments(struct hg_block *hb) +{ + long total = 0; + int i; + + for (i = 0; i < HG_STAT_SLOTS; i++) + total += hb->pstat[i].fragments; + return total < 0 ? 0 : (unsigned long)total; +} + +/* carved-but-idle cell capacity; defined in hg_arena.c (declared here too, + * since hg_arena.h includes THIS header and cannot be included back) */ +unsigned long hg_slab_recycled(struct hg_block *hb); + +/* + * Register HG_MALLOC's SHM arena statistics with the statistics collector, so + * the allocator's own state is scrapeable and not only reachable through the + * hg_stats MI command. Called from init_stats_collector(); a no-op unless + * HG_MALLOC is the shm allocator actually in use. Returns 0 on success. + * + * SHM ONLY, deliberately. Every allocator (qm/fm/hp/f_parallel/hg) reports + * through the same 7-field struct mem_info, and the core already turns that + * into shmem: plus per-process pkmem: and proc_ statistics - so HG's pkg memory + * is ALREADY visible by that generic route. No allocator exposes + * allocator-specific per-process statistics, and doing so would mean extending + * the signal_pkg_status()/pkg_status[][] mechanism, since a pkg arena is + * private memory that no other process can read. The shm arena has no such + * problem: there is exactly one, and it is shared. + */ +#ifdef HG_MALLOC +int hg_register_stats(void); +#else +/* No-op without the allocator, so callers (statistics.c) need no #ifdef of + * their own. Building with -DHG_MALLOC removed previously failed at link with + * an undefined reference here - the definition lives in mem/hg_arena.c, which + * is entirely inside "#ifdef HG_MALLOC". */ +static inline int hg_register_stats(void) { return 0; } +#endif + +/* bump a corruption counter; hb may be NULL where no arena is in scope */ +static inline void hg_corrupt(struct hg_block *hb, enum hg_corrupt_kind k) +{ + if (hb) + hb->corrupt[k]++; + else + hg_corrupt_noarena[k]++; +} + +static inline unsigned long hg_corrupt_total(struct hg_block *hb) +{ + unsigned long t = 0; + int i; + + for (i = 0; i < HG_CORRUPT_KINDS; i++) + t += hb->corrupt[i]; + return t; +} + +/* total cell-slot bytes handed out across every process */ +static inline unsigned long hg_cell_live(struct hg_block *hb) +{ + long total = 0; + int i; + + for (i = 0; i < HG_STAT_SLOTS; i++) + total += hb->pstat[i].cell_live; + return total < 0 ? 0 : (unsigned long)total; +} + +#ifdef STATISTICS +static inline unsigned long hg_get_size(struct hg_block *hb) +{ + return hb->size; +} +static inline unsigned long hg_get_used(struct hg_block *hb) +{ + return hg_used(hb); +} +/* These feed the shmem: and pkgmem: statistics (the SHM_GET_ and PKG_GET_ + * macros in shm_mem.h and mem.h) - a DIFFERENT path from hg_info(), so the + * recycled + * subtraction has to be applied here too or the published statistics still + * report the raw carve footprint. */ +static inline unsigned long hg_get_free(struct hg_block *hb) +{ + /* Headroom left to CARVE - deliberately NOT size minus the live figure. + * Carved-but-recycled cells are reusable only within their own size + * class, so counting them as free reports ~95% available right up until + * an allocation of a DIFFERENT size fails with "no more HG_MALLOC arena + * memory". hb->real_used is the carve footprint, so this is the number + * that actually predicts that failure. + * + * Consequence: free + real_used != size here (real_used is live), unlike + * q_malloc. The invariant that does hold is free + carve == size. Each + * figure answers a different question: used = live payload, real_used / + * max_used = live commitment and its peak, free = room left to carve. */ + return hb->size - hb->real_used; +} +/* The large tier's counterpart to hg_slab_recycled(): backing held from the + * buddy grid that is not currently handed out as a fragment. O(1), unlike the + * slab version, because both terms are maintained under hb->lock as the + * fragments come and go. */ +static inline unsigned long hg_large_recycled(struct hg_block *hb) +{ + return hb->large_backing > hb->large_live ? + hb->large_backing - hb->large_live : 0; +} +static inline unsigned long hg_get_real_used(struct hg_block *hb) +{ + /* real_used is now the true carve footprint of BOTH tiers - slab blocks + * plus whole large chunks - so both tiers' idle-but-held bytes have to + * come back out to leave what is genuinely handed out. */ + unsigned long recycled = hg_slab_recycled(hb) + hg_large_recycled(hb); + unsigned long live = hb->real_used > recycled ? + hb->real_used - recycled : 0; + + if (live > hb->max_live_used) + hb->max_live_used = live; + return live; +} +static inline unsigned long hg_get_max_real_used(struct hg_block *hb) +{ + /* refresh the mark first, so reading max on its own is not stale */ + (void)hg_get_real_used(hb); + return hb->max_live_used; +} +static inline unsigned long hg_get_frags(struct hg_block *hb) +{ + return hg_fragments(hb); +} +#endif /* STATISTICS */ + +#endif /* hg_malloc_h */ diff --git a/mem/hg_malloc_dyn.h b/mem/hg_malloc_dyn.h new file mode 100644 index 00000000000..ee29bb9562c --- /dev/null +++ b/mem/hg_malloc_dyn.h @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * Thin public-API layer over hg_cell_alloc()/hg_cell_free() (hg_arena.c). + * Much simpler than f_malloc_dyn.h/q_malloc_dyn.h's split_frag machinery: + * fixed size classes mean there is no fragment-splitting concept here at + * all - "does size X fit in cell class C" is a table lookup, not a runtime + * decision. Same ifdef-spaghetti convention as the other _dyn.h files + * though, for the same reasons (single/multi allocator x dbg/non-dbg). + */ + +#if !defined INLINE_ALLOC && defined DBG_MALLOC +void *hg_malloc_dbg(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line) +#elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC +void *hg_malloc(struct hg_block *hb, unsigned long size) +#else +void *hg_malloc(struct hg_block *hb, unsigned long size, + const char *file, const char *func, unsigned int line) +#endif +{ + void *p; + +#ifdef DBG_MALLOC + LM_GEN1(memlog, "%s_malloc(%lu), called from %s: %s(%d)\n", hb->name, + size, file, func, line); +#endif +#if HG_CELL_TAKES_DBG_ARGS + p = hg_cell_alloc(hb, size, file, func, line); +#else + p = hg_cell_alloc(hb, size); +#endif +#ifdef DBG_MALLOC + LM_GEN1(memlog, "%s_malloc(%lu), returns address %p\n", hb->name, + size, p); +#endif + return p; +} + +#if !defined INLINE_ALLOC && defined DBG_MALLOC +void hg_free_dbg(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line) +#elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC +void hg_free(struct hg_block *hb, void *p) +#else +void hg_free(struct hg_block *hb, void *p, const char *file, + const char *func, unsigned int line) +#endif +{ +#ifdef DBG_MALLOC + LM_GEN1(memlog, "%s_free(%p), called from %s: %s(%d)\n", hb->name, p, + file, func, line); + if (p && ((unsigned long)p < hb->lo || (unsigned long)p > hb->hi)) { + LM_CRIT("bad pointer %p (out of memory block!) - aborting\n", p); + abort(); + } +#endif +#if HG_CELL_TAKES_DBG_ARGS + hg_cell_free(hb, p, file, func, line); +#else + hg_cell_free(hb, p); +#endif +} + +#if !defined INLINE_ALLOC && defined DBG_MALLOC +void *hg_realloc_dbg(struct hg_block *hb, void *p, unsigned long size, + const char *file, const char *func, unsigned int line) +#elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC +void *hg_realloc(struct hg_block *hb, void *p, unsigned long size) +#else +void *hg_realloc(struct hg_block *hb, void *p, unsigned long size, + const char *file, const char *func, unsigned int line) +#endif +{ + unsigned char cls; + unsigned long cur_total = 0, need_total, cur_payload = 0, copy; + void *ptr; + +#ifdef DBG_MALLOC + LM_GEN1(memlog, "%s_realloc(%p, ->%lu), called from %s: %s(%d)\n", + hb->name, p, size, file, func, line); +#endif + + if (size == 0) { + if (p) + #if !defined INLINE_ALLOC && defined DBG_MALLOC + hg_free_dbg(hb, p, file, func, line); + #elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC + hg_free(hb, p); + #else + hg_free(hb, p, file, func, line); + #endif + return NULL; + } + + if (!p) + #if !defined INLINE_ALLOC && defined DBG_MALLOC + return hg_malloc_dbg(hb, size, file, func, line); + #elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC + return hg_malloc(hb, size); + #else + return hg_malloc(hb, size, file, func, line); + #endif + + /* fixed size classes: if the new size still fits the SAME class + * (accounting for the hidden header, same rule hg_cell_alloc() uses + * to pick a class), the existing cell is already big enough - no + * copy, no reallocation, unlike f_malloc's frag split/merge. + * + * Large objects (hg_large.c) take the simple path: always alloc new + + * copy + free old, no in-place grow/shrink via the boundary-tag + * frag's neighbors. That optimization is real future work, not + * needed for correctness - large reallocs are rarer still than large + * allocs to begin with. */ + /* validate ownership BEFORE reading the class - the old order read the + * header first and only then complained about a bad class, so a + * foreign or unmapped pointer faulted before reaching that check. */ + if (!hg_owns(hb, HG_HDR(p))) { + LM_CRIT("%s: realloc of %p, which is not from this arena " + "[%p,%p) - refusing it\n", hb->name, p, hb->hbase, + hb->hbase + hb->hsize); + return NULL; + } + + /* cur_payload is what the CALLER can legally read from @p, which is not + * cur_total minus one fixed header: the two tiers stack different + * headers in front of the payload. + * + * small : [ HG_CELL_HDR | payload ] + * large : [ HG_LFRAG_HDR_SIZE | HG_CELL_HDR | payload ] + * + * hg_large_frag_size() returns HG_LFRAG_HDR + frag->size (hg_large.c), + * i.e. the whole frag, and the large tier puts the cell header INSIDE + * that frag so the returned pointer looks like any other cell. Measured + * on the shipped geometry: a 70000-byte request yields frag_size 70072 + * = 32 + 40 + 70000. Subtracting only HG_CELL_HDR therefore over-counts + * by exactly HG_LFRAG_HDR_SIZE, on every ABI and every build config. */ + cls = HG_CLASS(p); + if (cls == HG_LARGE_MARKER) { + cur_total = hg_large_frag_size_at(HG_HDR(p) - HG_LFRAG_HDR_SIZE); + cur_payload = cur_total > HG_LFRAG_HDR_SIZE + HG_CELL_HDR ? + cur_total - HG_LFRAG_HDR_SIZE - HG_CELL_HDR : 0; + } else if (cls >= HG_NCLASSES) { + LM_CRIT("%s: cell %p carries invalid class %u - aborting " + "realloc\n", hb->name, p, cls); + return NULL; + } else { + cur_total = hg_cell_total_size(cls); + cur_payload = cur_total > HG_CELL_HDR ? cur_total - HG_CELL_HDR : 0; + need_total = ((size + HG_ROUNDTO - 1) / HG_ROUNDTO) * HG_ROUNDTO + HG_CELL_HDR; + if (need_total <= cur_total) + return p; + } + + #if !defined INLINE_ALLOC && defined DBG_MALLOC + ptr = hg_malloc_dbg(hb, size, file, func, line); + #elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC + ptr = hg_malloc(hb, size); + #else + ptr = hg_malloc(hb, size, file, func, line); + #endif + + if (ptr) { + /* realloc copies min(old, new) - never more than the caller had, + * and never more than the new allocation can hold. The missing + * clamp was the dangerous half: small cells are shielded by the + * need_total <= cur_total early return above, which catches every + * shrink, but the large branch has no such return, so a shrink + * from the large tier copied the WHOLE old payload into whatever + * the new (possibly thousand-fold smaller) allocation was. */ + copy = cur_payload < size ? cur_payload : size; + memcpy(ptr, p, copy); + #if !defined INLINE_ALLOC && defined DBG_MALLOC + hg_free_dbg(hb, p, file, func, line); + #elif !defined HG_MALLOC_DYN && !defined DBG_MALLOC + hg_free(hb, p); + #else + hg_free(hb, p, file, func, line); + #endif + } + + return ptr; +} + +#define HG_MALLOC_DYN diff --git a/mem/hg_version.h b/mem/hg_version.h new file mode 100644 index 00000000000..b8eb1c85044 --- /dev/null +++ b/mem/hg_version.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2026 VoIPcloud + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * The generation of the hugepage allocator this binary carries, as one + * string, used everywhere the allocator names itself: the compile-flag list + * in "opensips -V", the "allocator: ..." line each arena logs at startup, + * and mm_str(). + * + * It exists because those three places used to spell the name as three + * separate literals. A binary built from the v2 branch that still announced + * itself as plain HG_MALLOC would be indistinguishable from a v1 build in a + * log or a -V paste - which is exactly the evidence an A/B between the two + * arenas is judged on. + * + * Deliberately NOT the accepted spelling on the command line: -a HG_MALLOC + * keeps working (see parse_mm), because every /etc/default/opensips in the + * fleet passes that name and a build should never require the sizing file to + * be edited in lockstep. This is an identity, not a selector. + */ + +#ifndef HG_VERSION_H +#define HG_VERSION_H + +#define HG_MALLOC_NAME "HG_MALLOC_V3" + +#endif /* HG_VERSION_H */ diff --git a/mem/mem.c b/mem/mem.c index 16ce1d40cb5..ce1669f4533 100644 --- a/mem/mem.c +++ b/mem/mem.c @@ -80,13 +80,32 @@ int set_pkg_mm(const char *mm_name) int init_pkg_mallocs(void) { #ifdef PKG_MALLOC + /* + * HG_MALLOC does its OWN reservation (its own hugepage tier ladder, + * see hg_malloc_init() in hg_malloc.c) and never touches a + * pre-allocated address - unlike every other allocator here, which + * receives this malloc()'d block and lays its control structure out + * inside it. Calling malloc(pkg_mem_size) for HG_MALLOC would just + * leak that block (never freed, never used) - across dozens of + * worker processes that adds up, so the call is skipped for it + * rather than allocated-and-ignored. + * + * HG_INIT_INHERITED: this is the ONE arena every forked child + * inherits copy-on-write (pt.c hands each child a fresh arena of its + * own, but the parent's stays mapped in the child, holding all the + * module state parsed pre-fork). It must not sit on hugetlb - a + * child's COW fault there has no 4K fallback and no reservation, so + * an empty pool at fork is a silent SIGBUS. It starts on THP instead. + */ +#ifdef INLINE_ALLOC +#if defined HG_MALLOC + mem_block = hg_malloc_init(pkg_mem_size, "pkg", 0, NULL, HG_INIT_INHERITED); +#else mem_pool = malloc(pkg_mem_size); if (!mem_pool) { LM_CRIT("could not initialize PKG memory: %ld\n", pkg_mem_size); return -1; } - -#ifdef INLINE_ALLOC #if defined F_MALLOC mem_block = fm_malloc_init(mem_pool, pkg_mem_size, "pkg"); #elif defined Q_MALLOC @@ -94,10 +113,19 @@ int init_pkg_mallocs(void) #elif defined HP_MALLOC mem_block = hp_pkg_malloc_init(mem_pool, pkg_mem_size, "pkg"); #endif +#endif /* HG_MALLOC */ #else if (mem_allocator_pkg == MM_NONE) mem_allocator_pkg = mem_allocator; + if (mem_allocator_pkg != MM_HG_MALLOC && mem_allocator_pkg != MM_HG_MALLOC_DBG) { + mem_pool = malloc(pkg_mem_size); + if (!mem_pool) { + LM_CRIT("could not initialize PKG memory: %ld\n", pkg_mem_size); + return -1; + } + } + switch (mem_allocator_pkg) { #ifdef F_MALLOC case MM_F_MALLOC: @@ -147,6 +175,22 @@ int init_pkg_mallocs(void) gen_pkg_get_frags = (osips_get_mmstat_f)hp_pkg_get_frags; break; #endif +#ifdef HG_MALLOC + case MM_HG_MALLOC: + mem_block = hg_malloc_init(pkg_mem_size, "pkg", 0, NULL, HG_INIT_INHERITED); + gen_pkg_malloc = (osips_block_malloc_f)hg_malloc; + gen_pkg_realloc = (osips_block_realloc_f)hg_realloc; + gen_pkg_free = (osips_block_free_f)hg_free; + gen_pkg_info = (osips_mem_info_f)hg_info; + gen_pkg_status = (osips_mem_status_f)hg_status; + gen_pkg_get_size = (osips_get_mmstat_f)hg_get_size; + gen_pkg_get_used = (osips_get_mmstat_f)hg_get_used; + gen_pkg_get_rused = (osips_get_mmstat_f)hg_get_real_used; + gen_pkg_get_mused = (osips_get_mmstat_f)hg_get_max_real_used; + gen_pkg_get_free = (osips_get_mmstat_f)hg_get_free; + gen_pkg_get_frags = (osips_get_mmstat_f)hg_get_frags; + break; +#endif #ifdef DBG_MALLOC #ifdef F_MALLOC case MM_F_MALLOC_DBG: @@ -196,6 +240,22 @@ int init_pkg_mallocs(void) gen_pkg_get_frags = (osips_get_mmstat_f)hp_pkg_get_frags; break; #endif +#ifdef HG_MALLOC + case MM_HG_MALLOC_DBG: + mem_block = hg_malloc_init(pkg_mem_size, "pkg", 0, NULL, HG_INIT_INHERITED); + gen_pkg_malloc = (osips_block_malloc_f)hg_malloc_dbg; + gen_pkg_realloc = (osips_block_realloc_f)hg_realloc_dbg; + gen_pkg_free = (osips_block_free_f)hg_free_dbg; + gen_pkg_info = (osips_mem_info_f)hg_info; + gen_pkg_status = (osips_mem_status_f)hg_status_dbg; + gen_pkg_get_size = (osips_get_mmstat_f)hg_get_size; + gen_pkg_get_used = (osips_get_mmstat_f)hg_get_used; + gen_pkg_get_rused = (osips_get_mmstat_f)hg_get_real_used; + gen_pkg_get_mused = (osips_get_mmstat_f)hg_get_max_real_used; + gen_pkg_get_free = (osips_get_mmstat_f)hg_get_free; + gen_pkg_get_frags = (osips_get_mmstat_f)hg_get_frags; + break; +#endif #endif default: LM_ERR("current build does not include support for " diff --git a/mem/mem.h b/mem/mem.h index 30ec8cae06d..21c084a249b 100644 --- a/mem/mem.h +++ b/mem/mem.h @@ -113,6 +113,18 @@ extern unsigned long (*gen_pkg_get_frags)(void *blk); #define PKG_GET_MUSED() hp_pkg_get_max_real_used(mem_block) #define PKG_GET_FREE() hp_pkg_get_free(mem_block) #define PKG_GET_FRAGS() hp_pkg_get_frags(mem_block) +#elif defined HG_MALLOC +#define PKG_MALLOC_ hg_malloc +#define PKG_REALLOC hg_realloc +#define PKG_FREE hg_free +#define PKG_INFO hg_info +#define PKG_STATUS hg_status +#define PKG_GET_SIZE() hg_get_size(mem_block) +#define PKG_GET_USED() hg_get_used(mem_block) +#define PKG_GET_RUSED() hg_get_real_used(mem_block) +#define PKG_GET_MUSED() hg_get_max_real_used(mem_block) +#define PKG_GET_FREE() hg_get_free(mem_block) +#define PKG_GET_FRAGS() hg_get_frags(mem_block) #endif /* F_MALLOC */ #else #define PKG_MALLOC_ gen_pkg_malloc diff --git a/mem/shm_mem.c b/mem/shm_mem.c index 9d6d9fd9485..d73568ea091 100644 --- a/mem/shm_mem.c +++ b/mem/shm_mem.c @@ -274,7 +274,7 @@ void *shm_getmem(int fd, void *force_addr, unsigned long size) } -#if !defined(INLINE_ALLOC) && (defined(HP_MALLOC) || defined(F_PARALLEL_MALLOC)) +#if !defined(INLINE_ALLOC) && (defined(HP_MALLOC) || defined(F_PARALLEL_MALLOC) || defined(HG_MALLOC)) /* startup optimization */ int shm_use_global_lock = 1; #endif @@ -292,8 +292,13 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) shm_block = qm_malloc_init(mempool, pool_size, "shm"); #elif defined HP_MALLOC shm_block = hp_shm_malloc_init(mempool, pool_size, "shm"); -#elif define F_PARALEL_MALLOC +#elif defined F_PARALLEL_MALLOC shm_blocks[idx] = parallel_malloc_init(mempool, pool_size, "shm", idx); +#elif defined HG_MALLOC + /* ignores mempool/pool_size as an address - HG_MALLOC does its own + * reservation, pool_size is reused only as the size to request, see + * the shm_mem_init() caller-side skip of shm_getmem() for this case */ + shm_block = hg_malloc_init(pool_size, "shm", 1, NULL, 0); #endif #else @@ -318,6 +323,19 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) } #endif +#ifdef HG_MALLOC + if (mem_allocator_shm == MM_HG_MALLOC || mem_allocator_shm == MM_HG_MALLOC_DBG) { + /* HG_MALLOC does its own internal locking (hb->lock, slow-path + * only) - without this, shm_lock()/shm_unlock() would still + * acquire the unrelated F_MALLOC/Q_MALLOC-style global mem_lock + * around every hg_malloc()/hg_free() call: not a corruption risk + * (hg_malloc still does its own correct locking underneath), but + * it would serialize every shm allocation behind one lock again, + * defeating the fine-grained design entirely. */ + shm_use_global_lock = 0; + } +#endif + #ifdef SHM_EXTRA_STATS switch (mem_allocator_shm) { #ifdef F_MALLOC @@ -367,6 +385,18 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) shm_frag_func = parallel_frag_func; shm_frag_line = parallel_frag_line; break; +#endif +#ifdef HG_MALLOC + case MM_HG_MALLOC: + case MM_HG_MALLOC_DBG: + shm_stats_core_init = (osips_shm_stats_init_f)hg_stats_core_init; + shm_stats_get_index = hg_stats_get_index; + shm_stats_set_index = hg_stats_set_index; + shm_frag_overhead = HG_FRAG_OVERHEAD; + shm_frag_file = hg_frag_file; + shm_frag_func = hg_frag_func; + shm_frag_line = hg_frag_line; + break; #endif default: LM_ERR("current build does not include support for " @@ -399,6 +429,12 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) case MM_F_PARALLEL_MALLOC_DBG: shm_frag_size = parallel_frag_size; break; +#endif +#ifdef HG_MALLOC + case MM_HG_MALLOC: + case MM_HG_MALLOC_DBG: + shm_frag_size = hg_frag_size; + break; #endif default: LM_ERR("current build does not include support for " @@ -488,6 +524,27 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) gen_shm_get_frags = (osips_get_mmstat_f)hp_shm_get_frags; break; #endif +#ifdef HG_MALLOC + case MM_HG_MALLOC: + /* ignores mempool - HG_MALLOC does its own reservation; see the + * shm_mem_init() caller-side skip of shm_getmem() for this case */ + shm_block = hg_malloc_init(pool_size, "shm", 1, NULL, 0); + gen_shm_malloc = (osips_block_malloc_f)hg_malloc; + gen_shm_malloc_unsafe = (osips_block_malloc_f)hg_malloc; + gen_shm_realloc = (osips_block_realloc_f)hg_realloc; + gen_shm_realloc_unsafe = (osips_block_realloc_f)hg_realloc; + gen_shm_free = (osips_block_free_f)hg_free; + gen_shm_free_unsafe = (osips_block_free_f)hg_free; + gen_shm_info = (osips_mem_info_f)hg_info; + gen_shm_status = (osips_mem_status_f)hg_status; + gen_shm_get_size = (osips_get_mmstat_f)hg_get_size; + gen_shm_get_used = (osips_get_mmstat_f)hg_get_used; + gen_shm_get_rused = (osips_get_mmstat_f)hg_get_real_used; + gen_shm_get_mused = (osips_get_mmstat_f)hg_get_max_real_used; + gen_shm_get_free = (osips_get_mmstat_f)hg_get_free; + gen_shm_get_frags = (osips_get_mmstat_f)hg_get_frags; + break; +#endif #ifdef DBG_MALLOC #ifdef F_MALLOC case MM_F_MALLOC_DBG: @@ -546,6 +603,25 @@ int shm_mem_init_mallocs(void* mempool, unsigned long pool_size,int idx) gen_shm_get_frags = (osips_get_mmstat_f)hp_shm_get_frags; break; #endif +#ifdef HG_MALLOC + case MM_HG_MALLOC_DBG: + shm_block = hg_malloc_init(pool_size, "shm", 1, NULL, 0); + gen_shm_malloc = (osips_block_malloc_f)hg_malloc_dbg; + gen_shm_malloc_unsafe = (osips_block_malloc_f)hg_malloc_dbg; + gen_shm_realloc = (osips_block_realloc_f)hg_realloc_dbg; + gen_shm_realloc_unsafe = (osips_block_realloc_f)hg_realloc_dbg; + gen_shm_free = (osips_block_free_f)hg_free_dbg; + gen_shm_free_unsafe = (osips_block_free_f)hg_free_dbg; + gen_shm_info = (osips_mem_info_f)hg_info; + gen_shm_status = (osips_mem_status_f)hg_status_dbg; + gen_shm_get_size = (osips_get_mmstat_f)hg_get_size; + gen_shm_get_used = (osips_get_mmstat_f)hg_get_used; + gen_shm_get_rused = (osips_get_mmstat_f)hg_get_real_used; + gen_shm_get_mused = (osips_get_mmstat_f)hg_get_max_real_used; + gen_shm_get_free = (osips_get_mmstat_f)hg_get_free; + gen_shm_get_frags = (osips_get_mmstat_f)hg_get_frags; + break; +#endif #endif default: LM_ERR("current build does not include support for " @@ -730,6 +806,13 @@ int shm_dbg_mem_init_mallocs(void* mempool, unsigned long pool_size) case MM_HP_MALLOC_DBG: shm_dbg_block = hp_shm_malloc_init(mempool, pool_size, "shm_dbg"); break; +#endif +#ifdef HG_MALLOC + case MM_HG_MALLOC: + case MM_HG_MALLOC_DBG: + /* ignores mempool, same reasoning as shm_mem_init_mallocs() */ + shm_dbg_block = hg_malloc_init(pool_size, "shm_dbg", 1, NULL, 0); + break; #endif default: LM_ERR("current build does not include support for " @@ -831,6 +914,20 @@ int shm_mem_init(void) init_done = 1; return 0; +#ifdef HG_MALLOC + } else if (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG) { + /* HG_MALLOC does its own reservation (its own hugepage tier + * ladder) - shm_getmem()'s plain mmap is skipped entirely here, + * not just allocated-and-ignored, since for -m sized regions + * (often gigabytes) that would be a real leak, not a rounding + * error. mempool is passed as NULL: shm_mem_init_mallocs() never + * dereferences it for this allocator. */ +#ifndef USE_ANON_MMAP + close(fd); +#endif /* USE_ANON_MMAP */ + return shm_mem_init_mallocs(NULL, shm_mem_size, 0); +#endif } else { shm_mempool = shm_getmem(fd, NULL, shm_mem_size); #ifndef USE_ANON_MMAP @@ -847,6 +944,15 @@ int shm_mem_init(void) return shm_mem_init_mallocs(shm_mempool, shm_mem_size,0); } #else +#ifdef HG_MALLOC + if (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG) { +#ifndef USE_ANON_MMAP + close(fd); +#endif /* USE_ANON_MMAP */ + return shm_mem_init_mallocs(NULL, shm_mem_size, 0); + } +#endif shm_mempool = shm_getmem(fd, NULL, shm_mem_size); #ifndef USE_ANON_MMAP close(fd); @@ -883,6 +989,8 @@ int shm_dbg_mem_init(void) shm_dbg_pool_size = qm_get_dbg_pool_size(shm_memlog_size); #elif defined HP_MALLOC shm_dbg_pool_size = hp_get_dbg_pool_size(shm_memlog_size); + #elif defined HG_MALLOC + shm_dbg_pool_size = hg_get_dbg_pool_size(shm_memlog_size); #endif #else switch (mem_allocator_shm) { @@ -904,6 +1012,12 @@ int shm_dbg_mem_init(void) shm_dbg_pool_size = hp_get_dbg_pool_size(shm_memlog_size); break; #endif + #ifdef HG_MALLOC + case MM_HG_MALLOC: + case MM_HG_MALLOC_DBG: + shm_dbg_pool_size = hg_get_dbg_pool_size(shm_memlog_size); + break; + #endif default: LM_ERR("current build does not include support for " "selected allocator (%s)\n", mm_str(mem_allocator_shm)); @@ -915,6 +1029,17 @@ int shm_dbg_mem_init(void) LM_DBG("Debug SHM pool size: %.2lf GB\n", (double)shm_dbg_pool_size / 1024 / 1024 / 1024); + #ifdef HG_MALLOC + if (mem_allocator_shm == MM_HG_MALLOC || mem_allocator_shm == MM_HG_MALLOC_DBG) { + /* same reasoning as shm_mem_init(): skip shm_getmem()'s plain + * mmap entirely rather than allocate-and-ignore it */ + #ifndef USE_ANON_MMAP + close(fd_dbg); + #endif + return shm_dbg_mem_init_mallocs(NULL, shm_dbg_pool_size); + } + #endif + shm_dbg_mempool = shm_getmem(fd_dbg, NULL, shm_dbg_pool_size); #ifndef USE_ANON_MMAP @@ -972,6 +1097,12 @@ mi_response_t *mi_shm_check(const mi_params_t *params, int init_shm_post_yyparse(void) { +#ifdef HG_MALLOC + /* v3: resolve + attach the configured auto-scaling profiles - this is + * the first moment the config is known AND the arena already exists */ + if (hg_autoscale_post_cfg() < 0) + return -1; +#endif #ifdef HP_MALLOC if (mem_allocator_shm == MM_HP_MALLOC || mem_allocator_shm == MM_HP_MALLOC_DBG) { @@ -1086,6 +1217,31 @@ void shm_mem_destroy(void) return; #endif +#ifdef HG_MALLOC + /* + * shm_mempool/shm_dbg_mempool were never set for HG_MALLOC (see the + * shm_getmem() skip in shm_mem_init()/shm_dbg_mem_init()) - the real + * reservation is hb->hbase/hb->hsize, owned and released entirely by + * hg_malloc_destroy() (arena teardown + lock_destroy + munmap of the + * actual hugepage mapping). A runtime check, not a compile-time + * unconditional return like F_PARALLEL_MALLOC above: a multi-allocator + * build could have HG_MALLOC compiled in while some other allocator is + * the one actually selected at runtime, which still needs the cleanup + * below. + */ + if (mem_allocator_shm == MM_HG_MALLOC || mem_allocator_shm == MM_HG_MALLOC_DBG) { + if (shm_block) + hg_malloc_destroy((struct hg_block *)shm_block); + shm_block = NULL; +#ifdef DBG_MALLOC + if (shm_dbg_block) + hg_malloc_destroy((struct hg_block *)shm_dbg_block); + shm_dbg_block = NULL; +#endif + return; + } +#endif + #ifdef HP_MALLOC int j; diff --git a/mem/shm_mem.h b/mem/shm_mem.h index deaecc106a3..b08bc596638 100644 --- a/mem/shm_mem.h +++ b/mem/shm_mem.h @@ -82,6 +82,14 @@ #define shm_frag_file hp_frag_file #define shm_frag_func hp_frag_func #define shm_frag_line hp_frag_line +#elif defined HG_MALLOC +#define shm_stats_core_init hg_stats_core_init +#define shm_stats_get_index hg_stats_get_index +#define shm_stats_set_index hg_stats_set_index +#define shm_frag_overhead HG_FRAG_OVERHEAD +#define shm_frag_file hg_frag_file +#define shm_frag_func hg_frag_func +#define shm_frag_line hg_frag_line #endif #else extern void (*shm_stats_core_init)(void *blk, int core_index); @@ -103,6 +111,8 @@ extern unsigned long (*shm_frag_line)(void *p); #define shm_frag_size hp_frag_size #elif defined F_PARALLEL_MALLOC #define shm_frag_size parallel_frag_size +#elif defined HG_MALLOC +#define shm_frag_size hg_frag_size #endif #else extern unsigned long (*shm_frag_size)(void *p); @@ -188,7 +198,22 @@ extern unsigned long (*gen_shm_get_frags)(void *blk); #define SHM_GET_MUSED hp_shm_get_max_real_used #define SHM_GET_FREE hp_shm_get_free #define SHM_GET_FRAGS hp_shm_get_frags -#endif /* F_MALLOC || Q_MALLOC || HP_MALLOC */ +#elif defined HG_MALLOC +#define SHM_MALLOC hg_malloc +#define SHM_MALLOC_UNSAFE hg_malloc +#define SHM_REALLOC hg_realloc +#define SHM_REALLOC_UNSAFE hg_realloc +#define SHM_FREE hg_free +#define SHM_FREE_UNSAFE hg_free +#define SHM_INFO hg_info +#define SHM_STATUS hg_status +#define SHM_GET_SIZE hg_get_size +#define SHM_GET_USED hg_get_used +#define SHM_GET_RUSED hg_get_real_used +#define SHM_GET_MUSED hg_get_max_real_used +#define SHM_GET_FREE hg_get_free +#define SHM_GET_FRAGS hg_get_frags +#endif /* F_MALLOC || Q_MALLOC || HP_MALLOC || HG_MALLOC */ #else #define SHM_MALLOC gen_shm_malloc #define SHM_MALLOC_UNSAFE gen_shm_malloc_unsafe @@ -286,7 +311,7 @@ inline static void shm_threshold_check(void) #define shm_threshold_check() #endif -#if defined(HP_MALLOC) || defined(F_PARALLEL_MALLOC) +#if defined(HP_MALLOC) || defined(F_PARALLEL_MALLOC) || defined(HG_MALLOC) #ifdef INLINE_ALLOC #define shm_lock() #define shm_unlock() @@ -840,6 +865,22 @@ inline static void shm_info(struct mem_info* mi) inline static void shm_force_unlock(void) { +#ifdef HG_MALLOC + /* + * HG_MALLOC's lock lives inside the block itself (hb->lock), not as + * a separate global like mem_lock/mem_locks - its fast path holds no + * lock at all, only the slow path (chunk carve / gpool refill) briefly + * takes it, so a process crashing mid-slow-path can still wedge every + * other worker exactly like F_MALLOC's mem_lock can. Handled as its + * own early branch since it doesn't fit the mem_lock/mem_locks pattern + * the rest of this function is built around. + */ + if (mem_allocator_shm == MM_HG_MALLOC || mem_allocator_shm == MM_HG_MALLOC_DBG) { + if (shm_block) + lock_release(&((struct hg_block *)shm_block)->lock); + return; + } +#endif if (0 #if defined F_MALLOC || defined Q_MALLOC || mem_lock diff --git a/mi/mi_core.c b/mi/mi_core.c index ce493336722..c6497e6a469 100644 --- a/mi/mi_core.c +++ b/mi/mi_core.c @@ -44,6 +44,11 @@ #include "../net/net_tcp.h" #include "../mem/mem.h" #include "../mem/rpm_mem.h" +#ifdef HG_MALLOC +#include "../mem/shm_mem.h" +#include "../mem/hg_malloc.h" +#include "../core_stats.h" /* hg_pkg_peak_all: the per-process pkg high-water */ +#endif #include "../cachedb/cachedb.h" #include "../evi/event_interface.h" #include "../ipc.h" @@ -337,6 +342,11 @@ static mi_response_t *mi_ps(const mi_params_t *params, if (add_mi_string(proc_item, MI_SSTR("Type"), pt[i].desc, strlen(pt[i].desc)) < 0) goto error; + + /* -1 = never pinned (no pin_workers / no matching group) */ + if (add_mi_number(proc_item, MI_SSTR("PinnedCPU"), + pt[i].pinned_cpu) < 0) + goto error; } return resp; @@ -982,6 +992,571 @@ static mi_response_t *w_reload_routes(const mi_params_t *params, +#ifdef HG_MALLOC +/* + * HG_MALLOC keeps state the shared shmem:/pkgmem: statistics cannot express. + * Those six figures were designed for a free-list allocator, where freed + * memory returns to one general pool; HG_MALLOC instead CARVES the arena into + * fixed size-class chunks that are never given back, so "how much is + * committed", "how much is live" and "how much can still be handed out" stop + * being the same question. Rather than overload the shared names further, + * report the allocator's own view here. + */ +/* @per_process: this arena is private to the answering process (pkg), so stamp + * whose it is into the payload - see the call site for why that matters */ +static int hg_stats_one(mi_item_t *parent, char *name, struct hg_block *hb, + int per_process) +{ + mi_item_t *o, *cls_arr, *cls_item; + struct hg_chunk *ch; + unsigned int chunks_of[HG_NCLASSES], cell_size_of[HG_NCLASSES]; + unsigned long cells_of[HG_NCLASSES]; + unsigned long carved, live_committed; + const char *tier; + int c; + + if (!hb) + return 0; + + o = add_mi_object(parent, name, strlen(name)); + if (!o) + return -1; + + if (per_process) { + const char *d = (process_no >= 0 && pt) ? pt[process_no].desc : "?"; + + if (add_mi_number(o, MI_SSTR("pid"), my_pid()) < 0) + return -1; + if (add_mi_string(o, MI_SSTR("process"), (char *)d, strlen(d)) < 0) + return -1; + if (add_mi_string(o, MI_SSTR("scope"), + MI_SSTR("this process only - see pkmem: for every process")) < 0) + return -1; + } + + carved = hb->real_used; + tier = hg_mem_tier_str(hb->tier); + /* call this before reading max_live_used: the high-water mark is + * refreshed on read, so sampling it here keeps live_peak in step with + * shmem:max_used_size instead of reporting a stale 0 until something + * else happens to query the statistics */ + live_committed = hg_get_real_used(hb); + + if (add_mi_string(o, MI_SSTR("tier"), (char *)tier, strlen(tier)) < 0) + return -1; + /* + * "tier" above is what INIT achieved. Growth deltas negotiate their + * backing separately and may land lower, so a grown arena is described + * by the byte split, not by one label - the label alone would be the + * "outcome reported as an attribute" mistake. Emitted only when a + * second tier actually holds bytes, so the common case stays terse. + */ + { + int nt = 0, t; + for (t = HG_MEM_HUGETLB; t <= HG_MEM_4K; t++) + if (hb->tier_bytes[t]) + nt++; + if (nt > 1) { + mi_item_t *ta = add_mi_object(o, MI_SSTR("tier_bytes")); + + if (!ta) + return -1; + for (t = HG_MEM_HUGETLB; t <= HG_MEM_4K; t++) { + /* add_mi_number() takes a non-const name; the tier + * strings are literals it only reads */ + char *ts = (char *)hg_mem_tier_str((enum hg_mem_tier)t); + + if (hb->tier_bytes[t] && + add_mi_number(ta, ts, strlen(ts), + hb->tier_bytes[t]) < 0) + return -1; + } + } + } + if (add_mi_number(o, MI_SSTR("total_size"), hb->size) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("pinned_mb"), hb->locked_mb) < 0) + return -1; + /* v3: committed vs reserved, and whether growth has happened or been + * refused. committed == cap means fixed (v2 semantics, the default). */ + if (add_mi_number(o, MI_SSTR("committed"), hb->hsize) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("cap"), hb->hcap) < 0) + return -1; + if (hb->hcap > hb->hsize && + add_mi_number(o, MI_SSTR("grow_headroom"), hb->hcap - hb->hsize) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("grows"), hb->grows) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("grow_bytes"), hb->grow_bytes) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("grow_refused"), hb->grow_refused) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("grow_blocked"), hb->grow_blocked) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("shrinks"), hb->shrinks) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("shrink_bytes"), hb->shrink_bytes) < 0) + return -1; + + /* carved: bytes taken from the arena and cut into size-class chunks. + * Never returned - this is the figure that only ever grows, and the one + * that free_size counts down from. */ + if (add_mi_number(o, MI_SSTR("carved"), carved) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("carved_peak"), hb->max_real_used) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("chunks"), hb->nchunks) < 0) + return -1; + /* what shmem:free_size reports: arena never yet carved */ + if (add_mi_number(o, MI_SSTR("free_to_carve"), hb->size - carved) < 0) + return -1; + + /* live: what is actually handed out right now. live_committed and + * live_peak are what shmem:real_used_size and shmem:max_used_size + * report; payload and cells are shmem:used_size and shmem:fragments. */ + if (add_mi_number(o, MI_SSTR("live_committed"), live_committed) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("live_peak"), hb->max_live_used) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("live_payload"), hg_used(hb)) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("live_cells"), hg_fragments(hb)) < 0) + return -1; + /* SLAB ONLY - large allocations (> the top size class) are served by + * the boundary-tag tier and never appear here, so this is deliberately + * NOT comparable with live_payload above, which counts both. */ + if (add_mi_number(o, MI_SSTR("slab_cell_bytes_live"), + hg_cell_live(hb)) < 0) + return -1; + + /* carved but idle: on a private free stack or in the global pool. + * Reusable, but ONLY for its own size class - which is why it is not + * counted as free_to_carve. */ + /* + * v2 reclaim. blocks_carved counts every block ever cut and + * blocks_returned every one handed back, so the difference is the live + * block count and the ratio is how well reclaim keeps up. Watch + * carved against carved_peak too: under v1 carve was monotonic and the + * two were always equal, so carved BELOW its peak is itself the proof + * that memory is being given back. + */ + if (add_mi_number(o, MI_SSTR("blocks_carved"), hb->blocks_carved) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("blocks_returned"), hb->gc_blocks_returned) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("gc_passes"), hb->gc_passes) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("cache_flushes"), hb->cache_flushes) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("cells_flushed"), hb->cells_flushed) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("buddy_splits"), hb->buddy_splits) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("buddy_merges"), hb->buddy_merges) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("buddy_merges_init"), hb->buddy_merges_init) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("buddy_free_leaves"), hb->buddy_free_leaves) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("slab_recycled"), hg_slab_recycled(hb)) < 0) + return -1; + + /* + * Corruption detections. Zero is the only acceptable value: every site + * that bumps one of these has already refused an operation or leaked a + * cell. Broken out by kind because a recurrence of one defect looks very + * different from several unrelated rare ones, and the nfree_underflow + * kind in particular is the signature of the __thread palloc_slots bug + * family. Until now the ONLY detector was a log grep. + */ + { + static const char * const kind[HG_CORRUPT_KINDS] = { + "class_mismatch", "double_free", "nfree_underflow", + "bad_class", "foreign_ptr", "buddy_bad_free", "internal" + }; + mi_item_t *co; + int k; + + co = add_mi_object(o, MI_SSTR("corruption")); + if (!co) + return -1; + if (add_mi_number(co, MI_SSTR("total"), hg_corrupt_total(hb)) < 0) + return -1; + for (k = 0; k < HG_CORRUPT_KINDS; k++) + if (add_mi_number(co, (char *)kind[k], strlen(kind[k]), + hb->corrupt[k]) < 0) + return -1; + } + /* the large tier's own footprint, so the split between the two tiers + * inside carved is readable rather than inferred by subtraction */ + if (add_mi_number(o, MI_SSTR("large_backing"), hb->large_backing) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("large_live"), hb->large_live) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("large_recycled"), hg_large_recycled(hb)) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("large_chunks_carved"), + hb->large_chunks_carved) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("large_chunks_returned"), + hb->large_chunks_returned) < 0) + return -1; + /* + * Reserve-floor state. Without these the floor is unobservable in + * production: crossing it only emits one LM_WARN and bumps a sweep + * generation that is indistinguishable from the periodic sweep. + */ + if (add_mi_number(o, MI_SSTR("reserve_floor"), hb->reserve_floor) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("below_floor"), hb->below_floor) < 0) + return -1; + if (add_mi_number(o, MI_SSTR("floor_crossings"), hb->floor_crossings) < 0) + return -1; + + /* + * Per-order buddy free lists. buddy_free_leaves alone cannot say + * whether the free space is one contiguous run or the same number of + * leaves scattered at order 0 - which is the difference between an + * arena that can still serve a large chunk and one that cannot. + */ + { + mi_item_t *ord_arr, *ord_item; + unsigned int ord; + + ord_arr = add_mi_array(o, MI_SSTR("buddy_free_orders")); + if (!ord_arr) + return -1; + for (ord = 0; ord <= hb->buddy_top && ord <= HG_MAX_ORDERS; ord++) { + if (!hb->nfree[ord]) + continue; + ord_item = add_mi_object(ord_arr, 0, 0); + if (!ord_item) + return -1; + if (add_mi_number(ord_item, MI_SSTR("order"), ord) < 0) + return -1; + if (add_mi_number(ord_item, MI_SSTR("bytes"), + (unsigned long)HG_LEAF_SIZE << ord) < 0) + return -1; + if (add_mi_number(ord_item, MI_SSTR("blocks"), hb->nfree[ord]) < 0) + return -1; + } + } + + memset(chunks_of, 0, sizeof chunks_of); + memset(cell_size_of, 0, sizeof cell_size_of); + memset(cells_of, 0, sizeof cells_of); + for (ch = hb->chunks; ch; ch = ch->next) { + if (ch->cls >= HG_NCLASSES) + continue; + chunks_of[ch->cls]++; + cell_size_of[ch->cls] = ch->cell_size; + cells_of[ch->cls] += ch->cells; + } + + cls_arr = add_mi_array(o, MI_SSTR("classes")); + if (!cls_arr) + return -1; + for (c = 0; c < HG_NCLASSES; c++) { + if (!chunks_of[c]) + continue; + cls_item = add_mi_object(cls_arr, 0, 0); + if (!cls_item) + return -1; + if (add_mi_number(cls_item, MI_SSTR("cell_size"), + cell_size_of[c]) < 0) + return -1; + if (add_mi_number(cls_item, MI_SSTR("chunks"), chunks_of[c]) < 0) + return -1; + if (add_mi_number(cls_item, MI_SSTR("cells"), cells_of[c]) < 0) + return -1; + /* + * Free cells of this class in the SHARED pool. Cells sitting in + * a thread's private cache are not counted - they are unreachable + * from here by construction - so this is a lower bound on what is + * reusable. It is still the missing half of the picture: "cells" + * alone is capacity, and capacity cannot distinguish a class that + * is fully occupied from one that carved a lot and then went idle, + * which is precisely the shape of the 2026-08-10 wrong-class + * exhaustion on the staging SBCs. + */ + if (add_mi_number(cls_item, MI_SSTR("free_shared"), + hb->gpool_n[c]) < 0) + return -1; + } + + return 0; +} + +/* + * Sizing advice for -m / -M, from what the arenas have actually reached. + * + * Both margins are multipliers on the observed HIGH-WATER, not on current + * use: an arena that is 90% idle right now may still have peaked at 90% full + * during the busy hour, and it is the peak that has to fit. They differ + * because the two failure modes differ - shm exhaustion makes an allocation + * fail and a call drop, pkg exhaustion kills the process outright, so pkg + * gets the wider margin. + */ +#define HG_ADVISE_SHM_MARGIN 2 +#define HG_ADVISE_PKG_MARGIN 3 +/* + * Floors, deliberately low. An earlier 64 MB shm floor swallowed the answer: + * peak x margin came to ~13 MB on every workload tried, always below it, so + * the command replied "64" whatever it was asked and the arithmetic was never + * visible. A floor should stop a silly recommendation, not become the + * recommendation. + */ +#define HG_ADVISE_SHM_FLOOR_MB 8 +#define HG_ADVISE_PKG_FLOOR_MB 2 + +/* Bands. 80% was too late to be a warning: an arena at 77% of peak reported + * "reasonable" while being one busy hour from failing an allocation. */ +#define HG_ADVISE_TIGHT_PCT 70 +#define HG_ADVISE_WATCH_PCT 50 +#define HG_ADVISE_LOOSE_PCT 20 + +/* peak x margin, in MB, never below @floor, rounded up to a whole huge page + * so the arena does not map a partial one */ +static unsigned long hg_advise_mb(unsigned long peak, unsigned int margin, + unsigned long floor_mb, unsigned long hps) +{ + unsigned long mb = ((peak * margin) + (1UL << 20) - 1) >> 20; + unsigned long step = (hps >= (1UL << 20)) ? (hps >> 20) : 1; + + if (mb < floor_mb) + mb = floor_mb; + if (step > 1) + mb = ((mb + step - 1) / step) * step; + return mb; +} + +static const char *hg_advise_verdict(unsigned long peak, unsigned long size) +{ + unsigned long pct = size ? (peak * 100 / size) : 0; + + if (pct > HG_ADVISE_TIGHT_PCT) + return "TIGHT - raise it before the next busy period"; + if (pct > HG_ADVISE_WATCH_PCT) + return "watch - fine now, no room for a bad day"; + if (pct < HG_ADVISE_LOOSE_PCT) + return "oversized - the surplus is pinned and unusable elsewhere"; + return "reasonable"; +} + +static int hg_advise_one(mi_item_t *parent, const char *name, + struct hg_block *hb, unsigned long peak, unsigned int margin, + unsigned long floor_mb, int nproc) +{ + mi_item_t *o; + unsigned long rec; + + o = add_mi_object(parent, (char *)name, strlen(name)); + if (!o) + return -1; + + rec = hg_advise_mb(peak, margin, floor_mb, hb->hps); + + if (add_mi_number(o, MI_SSTR("configured_mb"), hb->size >> 20) < 0 || + add_mi_number(o, MI_SSTR("peak_bytes"), peak) < 0 || + add_mi_number(o, MI_SSTR("peak_pct_of_configured"), + hb->size ? (peak * 100 / hb->size) : 0) < 0 || + add_mi_number(o, MI_SSTR("margin_applied"), margin) < 0 || + add_mi_number(o, MI_SSTR("recommended_mb"), rec) < 0 || + add_mi_string(o, MI_SSTR("verdict"), + (char *)hg_advise_verdict(peak, hb->size), + strlen(hg_advise_verdict(peak, hb->size))) < 0) + return -1; + + /* + * Split the live figure into the part that is a property of the + * WORKLOAD and the part that is a property of the ARENA SIZE, because + * the second one makes any "shrink to N x observed" advice circular. + * + * Measured on an idle pkg arena across -M 32/16/8/4: slab live stayed + * at exactly 6,560 bytes while the large tier halved with every halving + * of -M. The cause is in reactor.c:85 - + * reactor_size = mem_size / n * FD_MEM_PERCENT / 100; + * the reactor's fd table is a PERCENTAGE of pkg memory, so it shrinks + * when the arena does. Apply a recommendation derived from total live + * and the next reading is smaller again, all the way to the floor. + * + * Reporting both halves is honest; guessing which large allocations are + * size-proportional and which are real workload is not, so no attempt + * is made to net it out automatically. + */ + { + unsigned long slab_live = hg_cell_live(hb); + unsigned long large_live = hb->large_live; + unsigned long live = slab_live + large_live; + unsigned long share = live ? (large_live * 100 / live) : 0; + + if (add_mi_number(o, MI_SSTR("live_slab_bytes"), slab_live) < 0 || + add_mi_number(o, MI_SSTR("live_large_bytes"), large_live) < 0 || + add_mi_number(o, MI_SSTR("large_share_pct"), share) < 0) + return -1; + + if (share > 50 && add_mi_string(o, MI_SSTR("caveat"), MI_SSTR( + "most of this arena's live bytes are in the large tier, and at " + "least some large consumers size themselves as a fraction of the " + "arena - so this recommendation is an upper bound, not a fixed " + "point. Apply it once, restart, and re-read rather than " + "iterating.")) < 0) + return -1; + } + + /* pkg is per-process, so the interesting number is the whole fleet of + * arenas, not one of them - that is what is pinned out of the hugepage + * pool and cannot be used by anything else */ + if (nproc > 0) { + if (add_mi_number(o, MI_SSTR("processes"), nproc) < 0 || + add_mi_number(o, MI_SSTR("pinned_total_mb"), + (unsigned long)nproc * (hb->size >> 20)) < 0 || + add_mi_number(o, MI_SSTR("recommended_total_mb"), + (unsigned long)nproc * rec) < 0) + return -1; + } + return 0; +} + +static mi_response_t *mi_hg_advise(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + mi_response_t *resp; + mi_item_t *resp_obj, *notes; + struct hg_block *hb; + unsigned long uptime = (unsigned long)(time(NULL) - startup_time); + int reported = 0; + + resp = init_mi_result_object(&resp_obj); + if (!resp) + return 0; + + if (add_mi_number(resp_obj, MI_SSTR("uptime_s"), uptime) < 0) + goto error; + + if (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG) { + hb = (struct hg_block *)shm_block; + /* the CARVE high-water, not the live one: carve is what has to + * fit in the arena, and in v2 it can sit well above live */ + if (hg_advise_one(resp_obj, "shm", hb, hb->max_real_used, + HG_ADVISE_SHM_MARGIN, HG_ADVISE_SHM_FLOOR_MB, 0) < 0) + goto error; + reported++; + } + + if (mem_allocator_pkg == MM_HG_MALLOC || + mem_allocator_pkg == MM_HG_MALLOC_DBG) { + unsigned long peak = 0, sum = 0; + int nproc = 0; + + hb = (struct hg_block *)mem_block; +#ifdef PKG_MALLOC + if (hg_pkg_peak_all(&peak, &sum, &nproc) < 0) + nproc = 0; +#endif + /* fall back to this process's own arena if the shared array is + * not populated - better a narrow answer than none, but say so + * in the notes below */ + if (!nproc) + peak = hb->max_real_used; + + if (hg_advise_one(resp_obj, "pkg", hb, peak, + HG_ADVISE_PKG_MARGIN, HG_ADVISE_PKG_FLOOR_MB, nproc) < 0) + goto error; + reported++; + } + + if (!reported) { + free_mi_response(resp); + return init_mi_error(400, + MI_SSTR("HG_MALLOC is not the active allocator")); + } + + notes = add_mi_array(resp_obj, MI_SSTR("notes")); + if (!notes) + goto error; + if (add_mi_string(notes, 0, 0, MI_SSTR( + "advice is based on the high-water reached SO FAR; a longer or " + "heavier window can only raise it")) < 0) + goto error; + if (uptime < 7200 && add_mi_string(notes, 0, 0, MI_SSTR( + "uptime is under 2 hours - too short to have seen a busy period, " + "treat the numbers as provisional")) < 0) + goto error; + if (add_mi_string(notes, 0, 0, MI_SSTR( + "pkg peak is the worst single process, which is the one -M has to " + "cover; every process gets its own arena of that size")) < 0) + goto error; + + return resp; + +error: + LM_ERR("failed to add mi item\n"); + free_mi_response(resp); + return 0; +} + +static mi_response_t *mi_hg_stats(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + mi_response_t *resp; + mi_item_t *resp_obj; + int reported = 0; + + resp = init_mi_result_object(&resp_obj); + if (!resp) + return 0; + + if (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG) { + if (hg_stats_one(resp_obj, "shm", (struct hg_block *)shm_block, 0) < 0) + goto error; + reported++; + } + + if (mem_allocator_pkg == MM_HG_MALLOC || + mem_allocator_pkg == MM_HG_MALLOC_DBG) { + /* + * pkg arenas are per-process, and the payload now says so rather + * than leaving it to a comment here. Over mi_fifo the answering + * process is the FIFO listener, which does no SIP work at all - + * reading its arena as "the" pkg arena is exactly how a worker + * running out of pkg stays invisible. The all-process picture is + * in the generic pkmem: statistics group. + */ + if (hg_stats_one(resp_obj, "pkg", (struct hg_block *)mem_block, 1) < 0) + goto error; + reported++; + } + + if (!reported) { + free_mi_response(resp); + return init_mi_error(400, + MI_SSTR("HG_MALLOC is not the active allocator")); + } + + /* + * Per-process, like the pkg figures above: frees this process handed + * back to an arena other than the one named by the caller. The name + * says so, because read over mi_fifo this only ever samples the FIFO + * listener and will sit at 0 no matter what the SIP workers do - a + * zero here is not evidence of anything. + */ + if (add_mi_number(resp_obj, MI_SSTR("cross_arena_frees_this_proc"), + hg_xarena_frees) < 0) + goto error; + + return resp; + +error: + LM_ERR("failed to add mi item\n"); + free_mi_response(resp); + return 0; +} +#endif /* HG_MALLOC */ + static const mi_export_t mi_core_cmds[] = { { "pi_list", "lists the provisioning framework", 0, 0, { {w_mi_pi_list, {0}}, @@ -1005,6 +1580,20 @@ static const mi_export_t mi_core_cmds[] = { {mi_pwd, {0}}, {EMPTY_MI_RECIPE}}, {0} }, +#ifdef HG_MALLOC + { "hg_stats", "HG_MALLOC arena internals: how much of the arena is " + "carved into size-class chunks, how much of that is live versus " + "recycled, and the per-class chunk breakdown", 0, 0, { + {mi_hg_stats, {0}}, + {EMPTY_MI_RECIPE}}, {0} + }, + { "hg_advise", "what -m and -M should be set to, derived from the " + "high-water each arena has actually reached; pkg advice covers the " + "worst single process, not the one answering", 0, 0, { + {mi_hg_advise, {0}}, + {EMPTY_MI_RECIPE}}, {0} + }, +#endif { "arg", "returns the full list of arguments used at startup", 0, 0, { {mi_arg, {0}}, {EMPTY_MI_RECIPE}}, {0} diff --git a/modules/hgstress/Makefile b/modules/hgstress/Makefile new file mode 100644 index 00000000000..e6dcff2d6dd --- /dev/null +++ b/modules/hgstress/Makefile @@ -0,0 +1,9 @@ +# hgstress module makefile +# +# WARNING: do not run this directly, it should be run by the master Makefile + +include ../../Makefile.defs +auto_gen= +NAME=hgstress.so + +include ../../Makefile.modules diff --git a/modules/hgstress/hgstress.c b/modules/hgstress/hgstress.c new file mode 100644 index 00000000000..5fc2a588a7c --- /dev/null +++ b/modules/hgstress/hgstress.c @@ -0,0 +1,530 @@ +/* + * hgstress - multi-process shm allocator soak test + * + * Copyright (C) 2026 Yury Kirsanov + * + * This file is part of opensips, a free SIP server. + * + * opensips is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version + * + * opensips is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * A throwaway stress module (not for production) that hammers the *shared* + * memory allocator from every forked worker at once and detects the failure + * mode that unit-style tests miss: the same cell being handed to two + * different processes. + * + * Method - each worker, after fork: + * 1. allocates a block and stamps EVERY 8-byte word of it with a value + * derived from its own pid and the block's slot/size; + * 2. keeps the block live in a slot table and, on later passes, re-reads + * those words and checks they are still its own stamp. + * + * If two processes are ever given the same address, the second one's stamp + * overwrites the first's, and the first detects a foreign pid on its next + * verify pass. That is reported as a "torn" block - the direct, in-process + * signature of a double hand-out, rather than waiting for the eventual + * downstream segfault in unrelated code. + * + * Sizes deliberately span the small-cell size classes AND cross the 64K cap + * into the large/coalescing tier, so both allocator paths are exercised and + * blocks are freed in a scattered order to force coalescing and reuse. + * + * Load with "modparam(hgstress, ...)" tuning and watch for the per-worker + * PASS/FAIL notice; the run aborts startup with -1 on any failure so a + * scripted test can just check the exit status. + */ + +#include +#include +#include +#include +#include + +#include "../../sr_module.h" +#include "../../timer.h" +#include "../../mi/mi.h" +#include "../../dprint.h" +#include "../../mem/shm_mem.h" +#include "../../locking.h" + +#define HGS_MAX_SLOTS 4096 + +static int hgs_slots = 512; /* live blocks held per worker */ +static int hgs_iters = 200000; /* alloc/free ops per worker */ +static int hgs_verify = 2000; /* full verify sweep every N ops */ +static int hgs_large = 64; /* 1-in-N allocations exceed the cell cap */ +/* + * v3 growth driver. Each worker first allocates and HOLDS hold_mb MB of + * stamped 128-512K blocks before the churn starts, and keeps them stamped + * through it. Size workers x hold_mb past -m and a fixed arena MUST report + * allocation failures (the fail-first arm), while a capped arena MUST grow + * to absorb it - with every worker already forked, which is the exact + * cross-process-visibility property the v3 mechanism was chosen for. A + * worker that SIGSEGVs touching grown space is the failure signature of + * the broken (per-process page table) growth design. + */ +static int hgs_hold_mb = 0; /* 0 = the classic churn-only soak */ +/* + * Same driver for the PKG arena. pkg is MAP_PRIVATE and per-process, so + * there is no cross-process question to prove here - what this arm proves + * is that the growth path executes for the private arena at all: every + * worker grows its OWN arena past -M independently, and the stamps verify + * the grown pages hold data. (The pkg cap multiplies by nproc in real + * RAM - a rig concern too: 5 workers x the cap.) + */ +static int hgs_hold_pkg_mb = 0; +/* + * Second cycle, for the shrink rig: at again_s seconds after startup a + * TIMER job re-runs one hold/verify/free cycle of hold_mb - in the timer + * process, which is fine for shm (the arena is shared; any process can + * drive it). The window between the workers' child_init soak ending + * (holds freed) and this cycle is where the shrink gate finds its quiet + * ticks; the cycle then proves the punched range recommits and serves + * stamped data again. 0 = off. + */ +static int hgs_again_s = 0; + +struct hgs_shared { + gen_lock_t lock; + int workers; + int failed; + unsigned long long ops; + unsigned long long torn; + unsigned long long alloc_failed; /* shm_malloc refusals, all workers */ +}; + +static struct hgs_shared *shared; + +struct hgs_blk { + char *p; + unsigned long len; +}; + +/* the stamp is per (pid, slot) so a foreign writer is identifiable, and it is + * written to EVERY word - a partial overlap is caught as readily as a whole + * one. + * + * uint64_t, not unsigned long. The stamp packs the pid into the high 32 bits + * and a hash of the slot into the low 32, which needs a 64-bit type to exist + * at all: on ILP32 "(unsigned long)pid << 32" shifts a 32-bit value by 32, + * which is UNDEFINED BEHAVIOUR, not a truncation. Whatever gcc emitted, the + * pid half was gone - so every worker's stamp collapsed to the slot hash + * alone, the detector could no longer tell a foreign writer from its own + * data, and the "foreign pid" it printed came from a second UB shift in the + * diagnostic itself. On LP64, where unsigned long is already 64-bit, this + * changes nothing whatsoever. */ +static inline uint64_t hgs_stamp(int pid, int slot) +{ + return ((uint64_t)pid << 32) ^ (uint64_t)(slot * 2654435761u); +} + +static void hgs_fill(struct hgs_blk *b, int pid, int slot) +{ + uint64_t v = hgs_stamp(pid, slot), *w = (uint64_t *)b->p; + unsigned long i, n = b->len / sizeof(uint64_t); + + for (i = 0; i < n; i++) + w[i] = v; +} + +/* returns the number of corrupted words */ +static unsigned long hgs_check(struct hgs_blk *b, int pid, int slot) +{ + uint64_t v = hgs_stamp(pid, slot), *w = (uint64_t *)b->p; + unsigned long i, n = b->len / sizeof(uint64_t), bad = 0; + + for (i = 0; i < n; i++) + if (w[i] != v) { + if (bad == 0) + LM_CRIT("torn block: slot %d len %lu word %lu: " + "got 0x%" PRIx64 " want 0x%" PRIx64 + " (foreign pid %" PRIu64 ")\n", + slot, b->len, i, w[i], v, + w[i] >> 32); + bad++; + } + return bad; +} + +static unsigned long hgs_pick_size(unsigned int *seed) +{ + /* small: spread across the size classes, incl. the sub-64B and the + * class-boundary sizes; large: past the 64K cell cap */ + if (hgs_large > 0 && (rand_r(seed) % hgs_large) == 0) + return 65536 + (rand_r(seed) % (512 * 1024)); + return 8 + (rand_r(seed) % 8000); +} + +static int hgs_run(int rank) +{ + struct hgs_blk *blk, *hold = NULL; + unsigned int seed; + unsigned long torn = 0, ops = 0, alloc_failed = 0; + unsigned long held = 0; + int nhold = 0, maxhold = 0; + int i, slot, pid = getpid(); + + blk = pkg_malloc(hgs_slots * sizeof *blk); + if (!blk) { + LM_ERR("no pkg memory for the slot table\n"); + return -1; + } + memset(blk, 0, hgs_slots * sizeof *blk); + seed = (unsigned int)pid ^ (unsigned int)rank; + + /* + * Hold phase: pile up hold_mb MB of stamped 128-512K blocks and KEEP + * them. This is what pushes total demand past the initial arena while + * every worker is alive, so growth (or, in the fail-first arm, the + * exhaustion it replaces) happens under the exact conditions the + * mechanism is claimed to survive. The blocks stay in the verify + * sweeps: a stamp that survives in a page committed after this + * process forked is the pass criterion, a SIGSEGV here is the + * signature of growth that edited only the grower's page tables. + */ + if (hgs_hold_mb > 0) { + maxhold = hgs_hold_mb * 8 + 8; /* 128K min -> at most 8/MB */ + hold = pkg_malloc(maxhold * sizeof *hold); + if (!hold) { + LM_ERR("no pkg memory for the hold table\n"); + pkg_free(blk); + return -1; + } + memset(hold, 0, maxhold * sizeof *hold); + + while (held < (unsigned long)hgs_hold_mb << 20 && + nhold < maxhold) { + hold[nhold].len = (128 << 10) + + (rand_r(&seed) % (384 << 10)); + hold[nhold].p = shm_malloc(hold[nhold].len); + if (!hold[nhold].p) { + alloc_failed++; + hold[nhold].len = 0; + break; /* the arena is done growing or fixed */ + } + /* the hold table reuses the slot-stamp scheme, offset so + * a hold block and a churn block never share a stamp */ + hgs_fill(&hold[nhold], pid, HGS_MAX_SLOTS + nhold); + held += hold[nhold].len; + nhold++; + } + LM_NOTICE("hgstress worker %d (pid %d): holding %lu KB in %d " + "blocks (%s)\n", rank, pid, held >> 10, nhold, + alloc_failed ? "STOPPED by allocation failure" : "target met"); + } + + /* + * PKG hold: same idea, per-process arena. Allocate-and-stamp past -M, + * verify at the end, free. Runs before the shm churn so a pkg-side + * crash is attributable at a glance. + */ + if (hgs_hold_pkg_mb > 0) { + struct hgs_blk *ph; + unsigned long pheld = 0; + int np = 0, pmax = hgs_hold_pkg_mb * 8 + 8; + unsigned long ptorn = 0; + + ph = pkg_malloc(pmax * sizeof *ph); + if (!ph) { + LM_ERR("no pkg memory for the pkg-hold table\n"); + pkg_free(blk); + if (hold) + pkg_free(hold); + return -1; + } + memset(ph, 0, pmax * sizeof *ph); + while (pheld < (unsigned long)hgs_hold_pkg_mb << 20 && + np < pmax) { + ph[np].len = (128 << 10) + (rand_r(&seed) % (384 << 10)); + ph[np].p = pkg_malloc(ph[np].len); + if (!ph[np].p) { + alloc_failed++; + ph[np].len = 0; + break; + } + hgs_fill(&ph[np], pid, 2 * HGS_MAX_SLOTS + np); + pheld += ph[np].len; + np++; + } + for (i = 0; i < np; i++) { + ptorn += hgs_check(&ph[i], pid, 2 * HGS_MAX_SLOTS + i); + pkg_free(ph[i].p); + } + pkg_free(ph); + torn += ptorn; + LM_NOTICE("hgstress worker %d (pid %d): pkg hold %lu KB in %d " + "blocks, %lu torn (%s)\n", rank, pid, pheld >> 10, np, + ptorn, np && pheld >= (unsigned long)hgs_hold_pkg_mb << 20 + ? "target met" : "STOPPED by allocation failure"); + } + + for (i = 0; i < hgs_iters; i++) { + slot = rand_r(&seed) % hgs_slots; + + if (blk[slot].p) { + torn += hgs_check(&blk[slot], pid, slot); + shm_free(blk[slot].p); + blk[slot].p = NULL; + } else { + blk[slot].len = hgs_pick_size(&seed); + blk[slot].p = shm_malloc(blk[slot].len); + if (!blk[slot].p) { + LM_ERR("shm_malloc(%lu) failed at op %d - increase -m\n", + blk[slot].len, i); + alloc_failed++; + blk[slot].len = 0; + continue; + } + hgs_fill(&blk[slot], pid, slot); + } + ops++; + + /* full sweep: catches a foreign write to a block this worker is + * holding but has not touched recently - the common case, since a + * double hand-out is usually noticed long after it happened. + * The hold blocks are swept too - they are the ones living in + * grown pages. */ + if (hgs_verify > 0 && (i % hgs_verify) == 0) { + for (slot = 0; slot < hgs_slots; slot++) + if (blk[slot].p) + torn += hgs_check(&blk[slot], pid, slot); + for (slot = 0; slot < nhold; slot++) + if (hold[slot].p) + torn += hgs_check(&hold[slot], pid, + HGS_MAX_SLOTS + slot); + } + } + + for (slot = 0; slot < hgs_slots; slot++) + if (blk[slot].p) { + torn += hgs_check(&blk[slot], pid, slot); + shm_free(blk[slot].p); + } + pkg_free(blk); + + /* final verify + release of the held load - the stamps have now + * survived the whole churn in place */ + for (slot = 0; slot < nhold; slot++) + if (hold[slot].p) { + torn += hgs_check(&hold[slot], pid, + HGS_MAX_SLOTS + slot); + shm_free(hold[slot].p); + } + if (hold) + pkg_free(hold); + + lock_get(&shared->lock); + shared->workers++; + shared->ops += ops; + shared->torn += torn; + shared->alloc_failed += alloc_failed; + if (torn) + shared->failed = 1; + lock_release(&shared->lock); + + if (torn) { + LM_CRIT("hgstress worker %d (pid %d): FAIL - %lu torn words " + "over %lu ops\n", rank, pid, torn, ops); + return -1; + } + + LM_NOTICE("hgstress worker %d (pid %d): PASS - %lu ops, 0 torn, " + "%lu alloc failures\n", rank, pid, ops, alloc_failed); + return 0; +} + +static void hgs_again(unsigned int ticks, void *param) +{ + struct hgs_blk *hold; + unsigned long held = 0, torn = 0; + int n = 0, i, maxhold = hgs_hold_mb * 8 + 8, pid = getpid(); + unsigned int seed = (unsigned int)pid ^ 0xa9a1u; + + hold = pkg_malloc(maxhold * sizeof *hold); + if (!hold) { + LM_ERR("again-cycle: no pkg for the hold table\n"); + return; + } + memset(hold, 0, maxhold * sizeof *hold); + while (held < (unsigned long)hgs_hold_mb << 20 && n < maxhold) { + hold[n].len = (128 << 10) + (rand_r(&seed) % (384 << 10)); + hold[n].p = shm_malloc(hold[n].len); + if (!hold[n].p) + break; + hgs_fill(&hold[n], pid, 3 * HGS_MAX_SLOTS + n); + held += hold[n].len; + n++; + } + for (i = 0; i < n; i++) { + torn += hgs_check(&hold[i], pid, 3 * HGS_MAX_SLOTS + i); + shm_free(hold[i].p); + } + pkg_free(hold); + LM_NOTICE("hgstress AGAIN-CYCLE (pid %d): held %lu KB in %d blocks, " + "%lu torn -> %s\n", pid, held >> 10, n, torn, + torn == 0 && held >= (unsigned long)hgs_hold_mb << 20 + ? "PASS" : (torn ? "FAIL" : "SHORT")); +} + +/* + * MI-driven persistent hold, for the autoscaling rig: the child_init soak + * blocks the timer processes (no sweep ticks until every child finishes), + * so proving PROACTIVE growth needs load applied from a normal running + * process - an MI worker. hgs_hold allocates and parks stamped blocks + * until hgs_release; both verify stamps. + */ +#define HGS_MI_MAX 4096 +static struct hgs_blk hgs_mi_held[HGS_MI_MAX]; +static int hgs_mi_n; + +static mi_response_t *mi_hgs_hold(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + int mb, pid = getpid(); + unsigned int seed; + unsigned long want, held = 0; + + if (get_mi_int_param(params, "mb", &mb) < 0 || mb <= 0) + return init_mi_param_error(); + want = (unsigned long)mb << 20; + seed = (unsigned int)pid ^ 0x5eed; + + while (held < want && hgs_mi_n < HGS_MI_MAX) { + struct hgs_blk *b = &hgs_mi_held[hgs_mi_n]; + + b->len = (128 << 10) + (rand_r(&seed) % (384 << 10)); + b->p = shm_malloc(b->len); + if (!b->p) + break; + hgs_fill(b, pid, 5 * HGS_MAX_SLOTS + hgs_mi_n); + held += b->len; + hgs_mi_n++; + } + LM_NOTICE("hgstress MI-HOLD: +%lu KB (%d blocks total held)\n", + held >> 10, hgs_mi_n); + return init_mi_result_string(MI_SSTR("OK")); +} + +static mi_response_t *mi_hgs_release(const mi_params_t *params, + struct mi_handler *async_hdl) +{ + unsigned long torn = 0; + int i, pid = getpid(); + + for (i = 0; i < hgs_mi_n; i++) { + torn += hgs_check(&hgs_mi_held[i], pid, 5 * HGS_MAX_SLOTS + i); + shm_free(hgs_mi_held[i].p); + hgs_mi_held[i].p = NULL; + } + LM_NOTICE("hgstress MI-RELEASE: %d blocks freed, %lu torn\n", + hgs_mi_n, torn); + hgs_mi_n = 0; + return torn ? init_mi_error(500, MI_SSTR("torn")) : + init_mi_result_string(MI_SSTR("OK")); +} + +static const mi_export_t mi_cmds[] = { + { "hgs_hold", "allocate and park N MB of stamped shm", 0, 0, { + {mi_hgs_hold, {"mb", 0}}, + {EMPTY_MI_RECIPE}}, {0} + }, + { "hgs_release", "verify and free everything hgs_hold parked", 0, 0, { + {mi_hgs_release, {0}}, + {EMPTY_MI_RECIPE}}, {0} + }, + {EMPTY_MI_EXPORT}, +}; + +static int mod_init(void) +{ + if (hgs_slots > HGS_MAX_SLOTS) + hgs_slots = HGS_MAX_SLOTS; + + if (hgs_again_s > 0 && + register_timer("hgstress-again", hgs_again, NULL, + (unsigned int)hgs_again_s, + TIMER_FLAG_DELAY_ON_DELAY) < 0) { + LM_ERR("failed to register the again-cycle timer\n"); + return -1; + } + + shared = shm_malloc(sizeof *shared); + if (!shared) { + LM_ERR("no shm memory for the shared result block\n"); + return -1; + } + memset(shared, 0, sizeof *shared); + if (!lock_init(&shared->lock)) { + LM_ERR("failed to init the result lock\n"); + return -1; + } + + LM_NOTICE("hgstress armed: %d slots, %d iters, verify every %d, " + "1-in-%d large, hold %d MB/worker\n", + hgs_slots, hgs_iters, hgs_verify, hgs_large, hgs_hold_mb); + return 0; +} + +static int child_init(int rank) +{ + if (!is_worker_proc(rank)) + return 0; + return hgs_run(rank); +} + +static void mod_destroy(void) +{ + if (!shared) + return; + LM_NOTICE("hgstress TOTAL: %d workers, %llu ops, %llu torn, " + "%llu alloc failures -> %s\n", + shared->workers, shared->ops, shared->torn, + shared->alloc_failed, shared->failed ? "FAIL" : "PASS"); +} + +static const param_export_t params[] = { + { "slots", INT_PARAM, &hgs_slots }, + { "iters", INT_PARAM, &hgs_iters }, + { "verify", INT_PARAM, &hgs_verify }, + { "large", INT_PARAM, &hgs_large }, + { "hold_mb", INT_PARAM, &hgs_hold_mb }, + { "hold_pkg_mb", INT_PARAM, &hgs_hold_pkg_mb }, + { "again_s", INT_PARAM, &hgs_again_s }, + { 0, 0, 0 } +}; + +struct module_exports exports = { + "hgstress", + MOD_TYPE_DEFAULT, + MODULE_VERSION, + DEFAULT_DLFLAGS, + 0, /* load function */ + NULL, /* dependencies */ + NULL, /* exported functions */ + NULL, /* exported async functions */ + params, + NULL, /* exported statistics */ + mi_cmds, /* exported MI functions */ + NULL, /* exported pseudo-variables */ + NULL, /* exported transformations */ + NULL, /* extra processes */ + NULL, /* pre-init function */ + mod_init, + NULL, /* reply processing */ + mod_destroy, + child_init, + NULL /* reload confirm */ +}; diff --git a/net/net_tcp.c b/net/net_tcp.c index f089cd1d10e..c8b67b2b7b8 100644 --- a/net/net_tcp.c +++ b/net/net_tcp.c @@ -67,6 +67,15 @@ #include "trans.h" #include "net_tcp_dbg.h" +#if defined(HG_MALLOC) && !defined(INLINE_ALLOC) +/* mem/hg_arena.c - flush this thread's allocator cache if a sweep is due. + * Declared here rather than by including mem/hg_arena.h: that header + * redefines HG_ROUNDTO and pulls in mem/common.h, which changes what the + * thread_malloc/thread_free MACROS expand to in this file and corrupts the + * libc heap at startup. See the call site in the IO pool loop. */ +void hg_cache_flush_if_due(void); +#endif + struct struct_hist_list *con_hist; enum tcp_worker_state { STATE_INACTIVE=0, STATE_ACTIVE, STATE_DRAINING}; @@ -1264,6 +1273,26 @@ static void *tcp_thread_routine(void *arg) /* Reactor operations stay in TCP main; IO threads only run read/write * callbacks and notify completion back to the main thread. */ while (1) { +#if defined(HG_MALLOC) && !defined(INLINE_ALLOC) + /* + * Job boundary: no locks held, no job in flight. The allocator's + * idle sweep cannot reach this thread by IPC - it waits on the + * condition variable below rather than on a reactor - so it leaves + * a generation counter and we act on it here. + * + * Declared locally rather than by including mem/hg_arena.h. That + * include is what caused the glibc heap corruption this hook was + * first shipped with: it drags in mem/hg_malloc.h, which #undefs + * and redefines HG_ROUNDTO and pulls mem/common.h into this + * translation unit, and thread_malloc/thread_free here are MACROS + * whose expansion depends on what is in scope. Allocate with one + * form and free with the other and libc's allocator reports an + * "unaligned fastbin chunk" at startup. The network layer has no + * business importing the allocator's internal headers for one + * void(void) call. + */ + hg_cache_flush_if_due(); +#endif cond_lock(&tcp_write_queue->cond); while (!tcp_pool.stop && tcp_pool.task_head == NULL && tcp_write_queue->head == NULL) @@ -2627,6 +2656,10 @@ int tcp_start_listener(void) .proc_desc = "TCP main", .flags = 0, .type = TYPE_NONE, + .pin_group = TYPE_TCP, + /* TCP main is multithreaded - its IO pool defaults to one thread + * per online CPU - so it takes the whole group, not one CPU */ + .pin_whole_group = 1, }; if (tcp_disabled) diff --git a/net/net_udp.c b/net/net_udp.c index e60605cf535..fc46611bf1d 100644 --- a/net/net_udp.c +++ b/net/net_udp.c @@ -425,6 +425,7 @@ static int fork_dynamic_udp_process(void *si_filter) .proc_desc = "UDP receiver", .flags = OSS_PROC_DYNAMIC|OSS_PROC_NEEDS_SCRIPT, .type = TYPE_UDP, + .sock = si, }; if ((p_id=internal_fork(&ifp_udp_rcv))<0) { @@ -505,7 +506,7 @@ int udp_start_processes(int *chd_rank, int *startup_done) struct socket_info_full *sif; int p_id; int i,p; - const struct internal_fork_params ifp_udp_rcv = { + struct internal_fork_params ifp_udp_rcv = { .proc_desc = "UDP receiver", .flags = OSS_PROC_NEEDS_SCRIPT, .type = TYPE_UDP, @@ -528,6 +529,9 @@ int udp_start_processes(int *chd_rank, int *startup_done) "auto forking will not be possible\n", si->name.len, si->name.s); + /* so a listener naming pin_cpus confines its own workers */ + ifp_udp_rcv.sock = si; + for (i=0;iworkers;i++) { (*chd_rank)++; if ( (p_id=internal_fork(&ifp_udp_rcv))<0 ) { diff --git a/pt.c b/pt.c index d6f9c828c3f..43ea3743f21 100644 --- a/pt.c +++ b/pt.c @@ -19,11 +19,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +/* for cpu_set_t / CPU_SET / sched_setaffinity, used by the pin_*_cpu() + * helpers below. Defined before the first include and never #undef'd - + * undefining it after the fact is what broke the musl build in lib/url.c + * (see PR #4119). */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include #include #include #include #include +#include +#include #include "lib/dbg/profiling.h" #include "mem/shm_mem.h" @@ -81,6 +91,274 @@ void register_fork_handler(struct internal_fork_handler *h) hp->_next = h; }; +/* + * CPU pinning is Linux-only: cpu_set_t and sched_setaffinity() are a + * glibc/Linux interface. OpenSIPS also builds on the BSDs, Solaris and + * Darwin, which each spell this differently (FreeBSD has cpuset_t and + * cpuset_setaffinity, for instance), so the whole feature is compiled out + * elsewhere rather than guessed at. Configuring it there is reported once + * instead of silently doing nothing. + */ +#ifdef __OS_linux + +/* + * Per-process-type CPU groups. + * + * "pin_workers=1" turns pinning on; each of these then optionally confines + * one kind of process to a CPU list: + * + * pin_udp_cpus = "0-7" + * pin_tcp_cpus = "8-11" + * pin_timer_cpus = "12" + * + * A type with no list set may use every CPU the process is allowed. Note + * that bin and hep are not process types of their own - they are transport + * protocols carried by the ordinary UDP/TCP workers, so they follow + * whichever of those they run over. + */ +static cpu_set_t pin_type_set[TYPE_MODULE + 1]; +static char pin_type_has[TYPE_MODULE + 1]; +static int pin_spec_parsed; + +/* "0-7,12" -> set. Returns -1 on malformed input. */ +static int pin_parse_cpulist(const char *s, cpu_set_t *set) +{ + long a, b; + char *end; + + CPU_ZERO(set); + while (*s) { + while (*s == ' ' || *s == ',') s++; + if (!*s) + break; + a = strtol(s, &end, 10); + if (end == s || a < 0 || a >= CPU_SETSIZE) + return -1; + s = end; + b = a; + if (*s == '-') { + s++; + b = strtol(s, &end, 10); + if (end == s || b < a || b >= CPU_SETSIZE) + return -1; + s = end; + } + for (; a <= b; a++) + CPU_SET(a, set); + } + return 0; +} + +static void pin_set_group(enum process_type t, const char *list, + const char *name) +{ + if (!list) + return; + if (pin_parse_cpulist(list, &pin_type_set[t]) < 0 || + CPU_COUNT(&pin_type_set[t]) == 0) { + LM_ERR("pin_%s_cpus: bad or empty CPU list \"%s\"\n", name, list); + return; + } + pin_type_has[t] = 1; + LM_INFO("pinning %s processes to %d CPU(s)\n", name, + CPU_COUNT(&pin_type_set[t])); +} + +/* did the config name a CPU list for any process type at all? */ +static int pin_any_group(void) +{ + int t; + + for (t = 0; t <= TYPE_MODULE; t++) + if (pin_type_has[t]) + return 1; + + return 0; +} + +/* parsed once, lazily, in the parent before any fork */ +static void pin_parse_spec(void) +{ + pin_spec_parsed = 1; + pin_set_group(TYPE_UDP, pin_udp_cpus, "udp"); + pin_set_group(TYPE_TCP, pin_tcp_cpus, "tcp"); + pin_set_group(TYPE_TIMER, pin_timer_cpus, "timer"); + pin_set_group(TYPE_MODULE, pin_module_cpus, "module"); +} + +/* + * Choose the CPU a new process should be pinned to, or -1 for "do not pin". + * Runs in the PARENT, before fork. + * + * Two things matter here: + * + * - The candidate CPUs come from the set this process is ALREADY allowed to + * run on, read back with sched_getaffinity() rather than assumed from the + * machine's CPU count. Under a cgroup or cpuset - a container, a systemd + * slice - we may be confined to a subset, and picking a raw CPU number + * would either fail or quietly widen affinity past what the operator + * confined us to. Intersecting can only ever narrow. + * + * - The CPU is the least-occupied one, counted over the processes actually + * running right now. Deriving it from the process-table slot instead would + * look balanced at startup and drift badly afterwards: with auto-scaling, + * slots are freed and reused, so a recycled slot can land on a CPU that + * already has workers while another sits idle. + */ +static int pin_pick_cpu(enum process_type ptype, struct socket_info *sock) +{ + cpu_set_t allowed; + cpu_set_t sock_set; + int count[CPU_SETSIZE]; + int i, n, best = -1, best_load = 0; + + if (!pin_workers && !(sock && sock->pin_cpus)) + return -1; + + if (!pin_spec_parsed) + pin_parse_spec(); + + CPU_ZERO(&allowed); + if (sched_getaffinity(0, sizeof allowed, &allowed) != 0) { + LM_WARN("cannot read CPU affinity, leaving new process unpinned: %s\n", + strerror(errno)); + return -1; + } + + /* narrow to this process type's group, if the spec named one. The + * intersection keeps the cpuset guarantee: a group can only ever + * restrict further, never grant a CPU we were not already allowed. */ + if (sock && sock->pin_cpus) { + /* the listener named its own CPUs - more specific than the group */ + if (pin_parse_cpulist(sock->pin_cpus, &sock_set) < 0 || + CPU_COUNT(&sock_set) == 0) { + LM_ERR("pin_cpus: bad or empty CPU list \"%s\" on listener " + "%.*s - leaving its workers unpinned\n", sock->pin_cpus, + sock->name.len, sock->name.s); + return -1; + } + CPU_AND(&allowed, &allowed, &sock_set); + if (CPU_COUNT(&allowed) == 0) { + LM_WARN("pin_cpus on listener %.*s has no CPU in common with " + "the allowed set - leaving its workers unpinned\n", + sock->name.len, sock->name.s); + return -1; + } + } else if (ptype >= 0 && ptype <= TYPE_MODULE && pin_type_has[ptype]) { + CPU_AND(&allowed, &allowed, &pin_type_set[ptype]); + if (CPU_COUNT(&allowed) == 0) { + LM_WARN("pin_workers: group for this process type has no CPU " + "in common with the allowed set - leaving unpinned\n"); + return -1; + } + } else if (pin_any_group()) { + /* This process belongs to no named group while other groups do + * exist. Pinning it anyway would place it by occupancy across + * every CPU, including the ones a group was given precisely so + * that nothing else would run there - which is the opposite of + * what the operator asked for. Leave it to the scheduler. */ + return -1; + } + + n = CPU_COUNT(&allowed); + if (n < 1) + return -1; + + memset(count, 0, sizeof count); + for (i = 0; i < counted_max_processes; i++) { + int c = pt[i].pinned_cpu; + + if (c >= 0 && c < CPU_SETSIZE && is_process_running(i)) + count[c]++; + } + + for (i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + if (best < 0 || count[i] < best_load) { + best = i; + best_load = count[i]; + } + } + + return best; +} + +/* Apply the choice made above. Runs in the CHILD - BEFORE process_no is + * set for this child (see the call site), so process_no here still reads + * as the parent's (always 0), not this child's real slot. Take the + * description as a parameter instead of relying on that global. */ +static void pin_apply_cpu(int cpu, const char *proc_desc) +{ + cpu_set_t one; + + if (cpu < 0) + return; + + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof one, &one) != 0) { + LM_WARN("failed to pin \"%s\" to CPU %d: %s\n", + proc_desc, cpu, strerror(errno)); + return; + } + + LM_INFO("\"%s\" pinned to CPU %d\n", proc_desc, cpu); +} + +/* Confine a multithreaded process to its group's whole CPU list. Runs in + * the CHILD, before any of its threads exist, so they all inherit it. */ +static void pin_apply_group(enum process_type t, const char *proc_desc) +{ + cpu_set_t set; + + if (!pin_spec_parsed) + pin_parse_spec(); + if (t < 0 || t > TYPE_MODULE || !pin_type_has[t]) + return; + + if (sched_getaffinity(0, sizeof set, &set) != 0) + return; + CPU_AND(&set, &set, &pin_type_set[t]); + if (CPU_COUNT(&set) == 0) { + LM_WARN("pin group for \"%s\" has no CPU in common with the " + "allowed set - leaving it unpinned\n", proc_desc); + return; + } + if (sched_setaffinity(0, sizeof set, &set) != 0) { + LM_WARN("failed to pin \"%s\" to its CPU group: %s\n", + proc_desc, strerror(errno)); + return; + } + LM_INFO("\"%s\" pinned to a %d-CPU group\n", proc_desc, + CPU_COUNT(&set)); +} + + +#else /* !__OS_linux */ + +static int pin_pick_cpu(enum process_type ptype, struct socket_info *sock) +{ + static int warned; + + if (pin_workers && !warned) { + warned = 1; + LM_WARN("CPU pinning is only implemented on Linux - " + "pin_workers and pin_*_cpus have no effect here\n"); + } + return -1; +} + +static void pin_apply_cpu(int cpu, const char *proc_desc) +{ +} + +static void pin_apply_group(enum process_type t, const char *proc_desc) +{ +} + +#endif /* __OS_linux */ + static unsigned long count_running_processes(void *x) { int i,cnt=0; @@ -119,6 +397,7 @@ int init_multi_proc_support(void) for( i=0 ; iflags & OSS_PROC_NEEDS_SCRIPT) && sroutes) { +#if defined(PKG_MALLOC) && defined(HG_MALLOC) + /* + * Under HG_MALLOC this child is about to abandon the whole pkg + * arena it inherited from the parent (internal_fork() swaps in + * a fresh, private one right after this handler chain), so + * freeing the route AST cell by cell buys nothing - and it was + * the one thing writing into the inherited arena before the + * swap: every such write is a copy-on-write fault against the + * parent's mapping, and while that mapping was hugetlb-backed + * an empty pool turned the very first one into a SIGBUS (how + * TCP main, the last no-script child, died at startup on a + * short pool). The parent's arena no longer sits on hugetlb + * (HG_INIT_INHERITED), so this is not load-bearing for safety + * any more - it just spares every no-script child a private + * 4K copy of each AST page it would otherwise dirty. Dropping + * the pointer is all the child needs. + */ + if (mem_allocator_pkg == MM_HG_MALLOC || + mem_allocator_pkg == MM_HG_MALLOC_DBG) { + sroutes = NULL; + return 0; + } +#endif free_route_lists(sroutes); sroutes = NULL; } @@ -320,6 +633,15 @@ int internal_fork(const struct internal_fork_params *ifpp) atomic_init(&pt[new_idx].startup_result, CHLD_STARTING); + /* decided here, in the parent, while the process table is stable; + * a whole-group process gets no single CPU - it is confined to the + * full group in the child instead, and must not count as occupying + * one slot of it here */ + pt[new_idx].pinned_cpu = ifpp->pin_whole_group ? -1 : + pin_pick_cpu(ifpp->pin_group ? + ifpp->pin_group : ifpp->type, + ifpp->sock); + if ( (pid=fork())<0 ){ LM_CRIT("cannot fork \"%s\" process (%d: %s)\n",ifpp->proc_desc, errno, strerror(errno)); @@ -331,6 +653,37 @@ int internal_fork(const struct internal_fork_params *ifpp) const struct internal_fork_handler *cfhp; /* child process */ is_main = 0; /* a child is not main process */ + + /* Pin BEFORE the allocator reset below: the reset makes this + * worker carve fresh chunks on first use, and we want that to + * happen once it is already on its final CPU. The CPU itself was + * chosen by the parent (pin_pick_cpu) so the decision could see a + * consistent view of who is running where. */ + if (ifpp->pin_whole_group) + pin_apply_group(ifpp->pin_group ? ifpp->pin_group : ifpp->type, + ifpp->proc_desc); + else + pin_apply_cpu(pt[new_idx].pinned_cpu, ifpp->proc_desc); + +#ifdef HG_MALLOC + /* + * MUST run before this child allocates anything. HG_MALLOC keeps + * its fast-path allocation state (per-size-class bump pointer + + * private free stack) in plain process memory, so a fresh child + * inherits an identical COPY of the parent's - pointing at the + * very same shm cells. Left alone, every worker would hand out + * the same cells to different callers. + */ + if (mem_allocator_shm == MM_HG_MALLOC || + mem_allocator_shm == MM_HG_MALLOC_DBG) { + hg_malloc_child_init((struct hg_block *)shm_block); +#ifdef DBG_MALLOC + if (shm_dbg_block) + hg_malloc_child_init((struct hg_block *)shm_dbg_block); +#endif + } +#endif /* HG_MALLOC */ + /* set uid */ process_no = new_idx; /* set attributes, pid etc */ @@ -364,6 +717,65 @@ int internal_fork(const struct internal_fork_params *ifpp) child_startup_failed(); } } +#if defined(PKG_MALLOC) && defined(HG_MALLOC) + /* + * HG_MALLOC as well as PKG_MALLOC: the body calls hg_malloc_init() + * and names struct hg_block, neither of which exists when the + * allocator is not compiled in. PKG_MALLOC alone is not enough - + * a build with pkg on and HG off failed here with an implicit + * declaration, which is how this was found. mem_allocator_pkg can + * never hold MM_HG_MALLOC in such a build (parse_mm rejects the + * name), so the runtime test below is unreachable there anyway. + */ + if (mem_allocator_pkg == MM_HG_MALLOC || + mem_allocator_pkg == MM_HG_MALLOC_DBG) { + /* + * pkg memory is private (MAP_PRIVATE), not shared like shm - so + * unlike the shm case above, resetting just the fast-path + * bookkeeping on the COW-inherited block is not enough: the + * block's own metadata (chunk list, bump offset, lock) lives + * INSIDE that same COW-shared region, so every process's + * writes to it silently diverge into disconnected private + * copies while all still pulling physical hugepages from the + * ONE shared reservation the parent made pre-fork - with + * enough worker processes this exhausts the hugetlb pool well + * beyond what pkg_mem_size alone would suggest, and confuses + * the kernel's own hugetlb reservation accounting. + * + * Give this child its own independent reservation instead of + * inheriting the parent's: hg_malloc_init() already knows how + * to try tier 1 (MAP_HUGETLB) and gracefully fall back through + * the tier ladder if the pool doesn't have enough left for this + * specific process - exactly the semantics wanted here, just + * never previously invoked per child. + * + * Runs after the post-fork handler chain, so anything a handler + * frees is still resolved against the arena it was allocated + * from (the inherited one). Two guarantees back this up now, + * neither of which depends on the hugetlb pool having a page + * left at fork time: + * - internal_fork_child_setup() no longer walks the route AST + * with pkg_free under HG (see there): core makes NO writes + * into the inherited arena before the swap; + * - the inherited arena is not hugetlb-backed + * (HG_INIT_INHERITED in mem.c): should anything ever write + * into it - now or later in the child's life - the COW is + * an ordinary page fault with a 4K fallback, not a SIGBUS. + * The parent's mapping stays in this child, unreferenced; the + * child reads parent-parsed module state through it and never + * allocates from it again. + */ + struct hg_block *child_pkg = + hg_malloc_init(pkg_mem_size, "pkg", 0, ifpp->proc_desc, 0); + if (!child_pkg) { + LM_CRIT("failed to init this child's own pkg memory " + "(%lu bytes)\n", pkg_mem_size); + exit(-1); + } + mem_block = child_pkg; + } +#endif + atomic_store(&pt[process_no].startup_result, CHLD_OK); return 0; }else{ diff --git a/pt.h b/pt.h index 1afc9b6d0d5..af63a735d19 100644 --- a/pt.h +++ b/pt.h @@ -98,6 +98,12 @@ struct process_table { /* the load statistic of this process */ struct proc_load_info load; + /* CPU this process is pinned to, or -1 if unpinned. Chosen by the + * parent at fork time (see pin_pick_cpu() in pt.c) and applied by the + * child; also what makes the choice occupancy-aware, since the parent + * counts the CPUs of the currently-running processes here. */ + int pinned_cpu; + /* synchronization during fork */ atomic_t startup_result; }; @@ -125,10 +131,28 @@ int count_child_processes(void); #define is_process_running(_idx) \ ( (pt[_idx].flags&OSS_PROC_IS_RUNNING)?1:0 ) +struct socket_info; + struct internal_fork_params { const char *proc_desc; unsigned int flags; enum process_type type; + /* The listener this process serves, when it serves one. A UDP + * listener may name its own CPU list (pin_cpus on the socket), which + * is more specific than the process-type group and wins over it. */ + struct socket_info *sock; + /* Which pin_*_cpus group this process belongs to, when that is not + * the same as its type. The core timer processes and the TCP manager + * are TYPE_NONE - they are not workers and take no script - yet an + * operator naming pin_timer_cpus or pin_tcp_cpus plainly means them + * too. Left at TYPE_NONE, the type is used. */ + enum process_type pin_group; + /* Pin this process to its group's WHOLE CPU list instead of one CPU + * chosen from it. For multithreaded processes - TCP main runs a pool + * of IO threads, one per online CPU by default - a single-CPU pin + * would serialise every thread onto that one core; the group is what + * the operator meant. Threads created after the pin inherit it. */ + int pin_whole_group; }; struct internal_fork_handler { diff --git a/socket_info.c b/socket_info.c index 650e4592049..3a59ba18fdb 100644 --- a/socket_info.c +++ b/socket_info.c @@ -183,12 +183,17 @@ struct socket_info_full* new_sock_info(struct socket_id *sid, str *orig_name) if (sid->workers) LM_WARN("number of workers per non UDP-based <%.*s> listener not " "supported -> ignoring...\n", si->name.len, si->name.s); + if (sid->pin_cpus) + LM_WARN("pin_cpus on non UDP-based <%.*s> listener not supported - " + "its workers come from one shared pool, not from this listener " + "-> ignoring...\n", si->name.len, si->name.s); if (sid->auto_scaling_profile) LM_WARN("auto-scaling for non UDP-based <%.*s> listener not " "supported -> ignoring...\n", si->name.len, si->name.s); } else { if (sid->workers) si->workers = sid->workers; + si->pin_cpus = sid->pin_cpus; si->tos = sid->tos; if (sid->auto_scaling_profile) { si->s_profile = get_scaling_profile(sid->auto_scaling_profile); diff --git a/socket_info.h b/socket_info.h index 012a14c525e..11516d2f161 100644 --- a/socket_info.h +++ b/socket_info.h @@ -67,6 +67,11 @@ struct socket_info { struct ip_addr adv_address; /* Advertised address in ip_addr form (for find_si) */ unsigned short adv_port; /* optimization for grep_sock_info() */ unsigned short workers; + /* CPU list this listener's workers are confined to, as written in + * the config ("0-3,8"); NULL means fall back to the process-type + * group. Only meaningful for the UDP-based protocols, which are the + * only ones with per-listener workers. */ + char *pin_cpus; unsigned short tos; struct scaling_profile *s_profile; void *extra_data; diff --git a/statistics.c b/statistics.c index 4516ba17b31..6652e149108 100644 --- a/statistics.c +++ b/statistics.c @@ -44,6 +44,7 @@ #include "dprint.h" #include "locking.h" #include "core_stats.h" +#include "mem/hg_malloc.h" #include "statistics.h" #include "pt.h" #include "globals.h" @@ -304,6 +305,14 @@ int init_stats_collector(void) goto error; } + /* HG_MALLOC's own shm-arena statistics - a no-op unless that allocator + * is in use. Registered here so they sit alongside the core memory + * statistics rather than behind a module that may not be loaded. */ + if (hg_register_stats() != 0) { + LM_ERR("failed to register the HG_MALLOC statistics\n"); + goto error; + } + /* register network-level statistics */ if (register_module_stats( "net", net_stats)!=0 ) { LM_ERR("failed to register network statistics\n"); diff --git a/timer.c b/timer.c index debacb297f5..9af431ec28a 100644 --- a/timer.c +++ b/timer.c @@ -706,11 +706,13 @@ int start_timer_processes(void) .proc_desc = "time_keeper", .flags = OSS_PROC_NO_IPC|OSS_PROC_NO_LOAD, .type = TYPE_NONE, + .pin_group = TYPE_TIMER, }, ifp_timer = { .proc_desc = "timer", .flags = OSS_PROC_NO_IPC|OSS_PROC_NO_LOAD, .type = TYPE_NONE, + .pin_group = TYPE_TIMER, }; /* diff --git a/version.h b/version.h index fee978f96f6..de80fbca7de 100644 --- a/version.h +++ b/version.h @@ -105,6 +105,13 @@ #define F_PARALLEL_MALLOC_STR "" #endif +#ifdef HG_MALLOC +#include "mem/hg_version.h" +#define HG_MALLOC_STR ", " HG_MALLOC_NAME +#else +#define HG_MALLOC_STR "" +#endif + #ifdef DBG_MALLOC #define DBG_MALLOC_STR ", DBG_MALLOC" #else @@ -187,7 +194,7 @@ STATS_STR EXTRA_STATS_STR EXTRA_DEBUG_STR \ DISABLE_NAGLE_STR USE_MCAST_STR NO_DEBUG_STR NO_LOG_STR \ SHM_MMAP_STR PKG_MALLOC_STR Q_MALLOC_STR F_MALLOC_STR \ - HP_MALLOC_STR F_PARALLEL_MALLOC_STR DBG_MALLOC_STR CC_O0_STR \ + HP_MALLOC_STR F_PARALLEL_MALLOC_STR HG_MALLOC_STR DBG_MALLOC_STR CC_O0_STR \ DEBUG_DMALLOC_STR QM_JOIN_FREE_STR FAST_LOCK_STR NOSMP_STR \ USE_PTHREAD_MUTEX_STR USE_UMUTEX_STR USE_POSIX_SEM_STR \ USE_SYSV_SEM_STR DBG_LOCK_STR