Skip to content

mem: HG_MALLOC - a hugepage slab allocator with elastic, auto-scaling arenas #4216

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Lt-Flash wants to merge 1 commit into
base: master
Choose a base branch
from
Open

mem: HG_MALLOC - a hugepage slab allocator with elastic, auto-scaling arenas #4216

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion Makefile.conf.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#db_oracle= Provides Oracle connectivity for OpenSIPS. | Development library of OCI, typically instantclient-sdk-10.2.0.3
#db_perlvdb= Provides a virtualization framework for OpenSIPS's database access. | Perl library development files, typically libperl-dev
#db_postgres= Provides Postgres connectivity for OpenSIPS | PostgreSQL library and development library - typically libpq5 and libpq-dev
#db_redis= Provides Redis (single instance or cluster) row storage for OpenSIPS | Redis client library, hiredis
#db_sqlite= Provides SQLite connectivity for OpenSIPS | SQLite library and development library - typically libsqlite3 and libsqlite3-dev
#db_unixodbc= Allows to use the unixodbc package with OpenSIPS | ODBC library and ODBC development library
#dialplan= Implements generic string translations based on matching and replacement rules | PCRE development library, typically libpcre2-dev
Expand Down Expand Up @@ -76,7 +77,7 @@
#uuid= UUID generator | uuid-dev

# Modules omitted from the default build, generally due to external dependencies.
exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_sqlite db_unixodbc dialplan emergency event_rabbitmq event_kafka event_sqs h350 httpd http2d identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dialoginfo presence_mwi presence_reginfo presence_xml presence_dfks proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python regex rabbitmq_consumer rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp
exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates clusterer_controller compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_redis db_sqlite db_unixodbc dialplan emergency event_kafka event_rabbitmq event_sqs h350 http2d httpd identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dfks presence_dialoginfo presence_mwi presence_reginfo presence_xml proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python rabbitmq_consumer regex rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp

# Modules forced into the build, even when also listed in exclude_modules.
include_modules?=
Expand All @@ -97,6 +98,17 @@ DEFS+= -DQ_MALLOC #Quality assurance memory allocator with runtime safety checks
DEFS+= -DHP_MALLOC #High performance allocator with fine-grained locking
DEFS+= -DDBG_MALLOC #Include additional, debug-enabled allocator flavors
DEFS+= -DF_PARALLEL_MALLOC #Fast memory allocator with paralel buckets
DEFS+= -DHG_MALLOC #Hugepage-backed slab allocator with a lock-free fast path
# HG_MALLOC reserves its arena up front on 2 MB huge pages, pre-faults and
# mlock()s it, and serves fixed size classes from per-thread caches so the
# common path takes no lock. A block belongs to one size class while in use;
# once it drains completely it returns to the class-agnostic buddy and can be
# re-carved for a different class, so the carve shrinks again after a burst.
# Because the arena is pinned, resident memory equals -m plus -M x processes
# from the first second - unlike the mmap-backed allocators, oversizing is not
# cheap. It needs a huge-page pool of at least (-m + -M x procs) / 2 pages and
# a raised memlock limit; without them it drops to a lower tier and keeps
# running. See mem/README.hg_malloc and mem/README.hg_arena_v2
#DEFS+= -DQM_DBG_MALLOC_HIST=3 #Size of debug history for Q_MALLOC (default is 1)
#DEFS+= -DNO_DEBUG #Compile out all debug messages
#DEFS+= -DNO_LOG #Compile out all logging
Expand Down
6 changes: 6 additions & 0 deletions Makefile.defs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,10 +524,12 @@ endif
ifeq (,$(findstring F_MALLOC, $(DEFS)))
ifeq (,$(findstring Q_MALLOC, $(DEFS)))
ifeq (,$(findstring HP_MALLOC, $(DEFS)))
ifeq (,$(findstring HG_MALLOC, $(DEFS)))
DEFS+= -DQ_MALLOC
endif
endif
endif
endif

ifneq (,$(MEM_STATS_HDR))
EXPECTED_MD5=$(shell [ -f $(MEM_STATS_HDR) ] && tail -n 1 $(MEM_STATS_HDR) | awk '{print $$4}')
Expand Down Expand Up @@ -558,6 +560,10 @@ MKTAGS=ctags -R .
# decently fast allocator with extra sanity checks + safety buffers
# -DHP_MALLOC
# high performance allocator optimized for shared memory multiprocessing
# -DHG_MALLOC
# hugepage-backed slab allocator; fixed size classes bump-carved from
# its own huge-page (or best-effort degraded) reservation, no fallback
# to another allocator on exhaustion - increase -m/-M instead
# -DDBG_MALLOC
# will cause each of the enabled allocators to also include a "debugging"
# flavor of itself; by running a debugging version of an allocator, you
Expand Down
24 changes: 24 additions & 0 deletions cfg.lex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,11 @@ RPM_MEM_FILE "restart_persistency_cache_file"
RPM_MEM_SIZE "restart_persistency_size"
MEMLOG "memlog"|"mem_log"
MEMDUMP "memdump"|"mem_dump"
PIN_WORKERS "pin_workers"|"cpu_pinning"
PIN_UDP_CPUS "pin_udp_cpus"
PIN_TCP_CPUS "pin_tcp_cpus"
PIN_TIMER_CPUS "pin_timer_cpus"
PIN_MODULE_CPUS "pin_module_cpus"
SHM_MEMLOG_SIZE "shm_memlog_size"
EXECMSGTHRESHOLD "execmsgthreshold"|"exec_msg_threshold"
WORKER_REACTOR_TIMEOUT "workerreactortimeout"|"worker_reactor_timeout"
Expand Down Expand Up @@ -290,6 +295,10 @@ DB_DEFAULT_URL "db_default_url"
DB_MAX_ASYNC_CONNECTIONS "db_max_async_connections"
DISABLE_503_TRANSLATION "disable_503_translation"
AUTO_SCALING_PROFILE "auto_scaling_profile"
SHM_AUTO_SCALING_PROFILE "shm_auto_scaling_profile"
PKG_AUTO_SCALING_PROFILE "pkg_auto_scaling_profile"
HG_RAM_FLOOR_MB "hg_ram_floor_mb"
HG_AUTOSCALE_DRY_RUN "hg_autoscale_dry_run"
AUTO_SCALING_CYCLE "auto_scaling_cycle"
TIMER_WORKERS "timer_workers"

Expand Down Expand Up @@ -319,6 +328,7 @@ TICK \'
SLASH "/"
AS {EAT_ABLE}("as"|"AS"){EAT_ABLE}
USE_WORKERS {EAT_ABLE}("use_workers"|"USE_WORKERS"){EAT_ABLE}
PIN_CPUS {EAT_ABLE}("pin_cpus"|"PIN_CPUS"){EAT_ABLE}
SOCK_TOS {EAT_ABLE}("tos"|"TOS"){EAT_ABLE}
USE_AUTO_SCALING_PROFILE {EAT_ABLE}("use_auto_scaling_profile"|"USE_AUTO_SCALING_PROFILE"){EAT_ABLE}
SCALE_UP_TO {EAT_ABLE}("scale"|"SCALE"){EAT_ABLE}+("up"|"UP"){EAT_ABLE}+("to"|"TO"){EAT_ABLE}
Expand Down Expand Up @@ -481,6 +491,11 @@ SPACE [ ]
<INITIAL>{RPM_MEM_SIZE} { count(); yylval.strval=yytext; return RPM_MEM_SIZE; }
<INITIAL>{MEMLOG} { count(); yylval.strval=yytext; return MEMLOG; }
<INITIAL>{MEMDUMP} { count(); yylval.strval=yytext; return MEMDUMP; }
<INITIAL>{PIN_WORKERS} { count(); yylval.strval=yytext; return PIN_WORKERS; }
<INITIAL>{PIN_UDP_CPUS} { count(); yylval.strval=yytext; return PIN_UDP_CPUS; }
<INITIAL>{PIN_TCP_CPUS} { count(); yylval.strval=yytext; return PIN_TCP_CPUS; }
<INITIAL>{PIN_TIMER_CPUS} { count(); yylval.strval=yytext; return PIN_TIMER_CPUS; }
<INITIAL>{PIN_MODULE_CPUS} { count(); yylval.strval=yytext; return PIN_MODULE_CPUS; }
<INITIAL>{SHM_MEMLOG_SIZE} { count(); yylval.strval=yytext; return SHM_MEMLOG_SIZE; }
<INITIAL>{EXECMSGTHRESHOLD} { count(); yylval.strval=yytext; return EXECMSGTHRESHOLD; }
<INITIAL>{WORKER_REACTOR_TIMEOUT} { count(); yylval.strval=yytext; return WORKER_REACTOR_TIMEOUT; }
Expand Down Expand Up @@ -544,6 +559,14 @@ SPACE [ ]
return DISABLE_503_TRANSLATION; }
<INITIAL>{AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext;
return AUTO_SCALING_PROFILE; }
<INITIAL>{SHM_AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext;
return SHM_AUTO_SCALING_PROFILE; }
<INITIAL>{PKG_AUTO_SCALING_PROFILE} { count(); yylval.strval=yytext;
return PKG_AUTO_SCALING_PROFILE; }
<INITIAL>{HG_RAM_FLOOR_MB} { count(); yylval.strval=yytext;
return HG_RAM_FLOOR_MB; }
<INITIAL>{HG_AUTOSCALE_DRY_RUN} { count(); yylval.strval=yytext;
return HG_AUTOSCALE_DRY_RUN; }
<INITIAL>{AUTO_SCALING_CYCLE} { count(); yylval.strval=yytext;
return AUTO_SCALING_CYCLE; }
<INITIAL>{TIMER_WORKERS} { count(); yylval.strval=yytext;
Expand Down Expand Up @@ -603,6 +626,7 @@ SPACE [ ]
<INITIAL>{COMMA} { count(); return COMMA; }
<INITIAL>{SEMICOLON} { count(); return SEMICOLON; }
<INITIAL>{USE_WORKERS} { count(); return USE_WORKERS; }
<INITIAL>{PIN_CPUS} { count(); return PIN_CPUS; }
<INITIAL>{SOCK_TOS} { count(); return SOCK_TOS; }
<INITIAL>{USE_AUTO_SCALING_PROFILE} { count(); return USE_AUTO_SCALING_PROFILE; }
<INITIAL>{COLON} { count(); return COLON; }
Expand Down
45 changes: 45 additions & 0 deletions cfg.y
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,7 @@ extern char *finame;
struct listen_param {
enum si_flags flags;
int workers;
char *pin_cpus;
int tos;
struct socket_id *socket;
char *tag;
Expand Down Expand Up @@ -353,6 +354,11 @@ extern int cfg_parse_only_routes;
%token RPM_MEM_SIZE
%token MEMLOG
%token MEMDUMP
%token PIN_WORKERS
%token PIN_UDP_CPUS
%token PIN_TCP_CPUS
%token PIN_TIMER_CPUS
%token PIN_MODULE_CPUS
%token SHM_MEMLOG_SIZE
%token EXECMSGTHRESHOLD
%token WORKER_REACTOR_TIMEOUT
Expand Down Expand Up @@ -407,6 +413,10 @@ extern int cfg_parse_only_routes;
%token ASYNC_TOKEN
%token LAUNCH_TOKEN
%token AUTO_SCALING_PROFILE
%token SHM_AUTO_SCALING_PROFILE
%token PKG_AUTO_SCALING_PROFILE
%token HG_RAM_FLOOR_MB
%token HG_AUTOSCALE_DRY_RUN
%token AUTO_SCALING_CYCLE
%token TIMER_WORKERS

Expand Down Expand Up @@ -461,6 +471,7 @@ extern int cfg_parse_only_routes;
%token SLASH
%token AS
%token USE_WORKERS
%token PIN_CPUS
%token SOCK_TOS
%token USE_AUTO_SCALING_PROFILE
%token MAX
Expand Down Expand Up @@ -763,6 +774,9 @@ socket_def_param: ANYCAST { IFOR();
| USE_WORKERS NUMBER { IFOR();
p_tmp.workers=$2;
}
| PIN_CPUS STRING { IFOR();
p_tmp.pin_cpus=$2;
}
| SOCK_TOS NUMBER { IFOR();
p_tmp.tos=$2;
}
Expand Down Expand Up @@ -1259,6 +1273,16 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR();
| SHM_MEMLOG_SIZE EQUAL error { yyerror("int value expected"); }
| MEMDUMP EQUAL snumber { IFOR(); memdump=$3; }
| MEMDUMP EQUAL error { yyerror("int value expected"); }
| PIN_WORKERS EQUAL snumber { IFOR(); pin_workers=$3; }
| PIN_WORKERS EQUAL error { yyerror("int value expected"); }
| PIN_UDP_CPUS EQUAL STRING { IFOR(); pin_udp_cpus=$3; pin_workers=1; }
| PIN_UDP_CPUS EQUAL error { yyerror("string value expected"); }
| PIN_TCP_CPUS EQUAL STRING { IFOR(); pin_tcp_cpus=$3; pin_workers=1; }
| PIN_TCP_CPUS EQUAL error { yyerror("string value expected"); }
| PIN_TIMER_CPUS EQUAL STRING { IFOR(); pin_timer_cpus=$3; pin_workers=1; }
| PIN_TIMER_CPUS EQUAL error { yyerror("string value expected"); }
| PIN_MODULE_CPUS EQUAL STRING { IFOR(); pin_module_cpus=$3; pin_workers=1; }
| PIN_MODULE_CPUS EQUAL error { yyerror("string value expected"); }
| EXECMSGTHRESHOLD EQUAL NUMBER { IFOR();execmsgthreshold=$3; }
| EXECMSGTHRESHOLD EQUAL error { yyerror("int value expected"); }
| WORKER_REACTOR_TIMEOUT EQUAL NUMBER { IFOR(); worker_reactor_timeout=$3; }
Expand Down Expand Up @@ -1644,6 +1668,26 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR();
| AUTO_SCALING_PROFILE EQUAL error {
yyerror("bad auto-scaling profile definition");
}
| SHM_AUTO_SCALING_PROFILE EQUAL ID { IFOR();
hg_shm_profile_name=$3; }
| SHM_AUTO_SCALING_PROFILE EQUAL error {
yyerror("profile name expected");
}
| PKG_AUTO_SCALING_PROFILE EQUAL ID { IFOR();
hg_pkg_profile_name=$3; }
| PKG_AUTO_SCALING_PROFILE EQUAL error {
yyerror("profile name expected");
}
| HG_RAM_FLOOR_MB EQUAL NUMBER { IFOR();
hg_ram_floor_mb=$3; }
| HG_RAM_FLOOR_MB EQUAL error {
yyerror("integer value expected");
}
| HG_AUTOSCALE_DRY_RUN EQUAL NUMBER { IFOR();
hg_autoscale_dry_run=$3; }
| HG_AUTOSCALE_DRY_RUN EQUAL error {
yyerror("integer value expected");
}
| AUTO_SCALING_CYCLE EQUAL NUMBER { IFOR();
auto_scaling_cycle=$3; }
| AUTO_SCALING_CYCLE EQUAL error {
Expand Down Expand Up @@ -2797,6 +2841,7 @@ static void fill_socket_id(struct listen_param *param, struct socket_id *s)
while (s) {
s->flags |= param->flags;
s->workers = param->workers;
s->pin_cpus = param->pin_cpus;
s->tos = param->tos;
s->auto_scaling_profile = param->auto_scaling_profile;
s->tag = param->tag;
Expand Down
Loading