From 8e8c0bfcbffbe98b19793f9048a4644c041be814 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Thu, 20 Aug 2026 20:56:48 +0800 Subject: [PATCH 1/2] fix(sdk): default llama_cpp thread counts to llama.cpp's own value The NPU/GPU device paths hardcoded n_threads/n_threads_batch to 6 on every platform, an unfulfilled compute_configs.py placeholder from the original per-(platform, device) params refactor. llama.cpp's own tools never vary thread count by device and default to common_cpu_get_num_math() for both decode and batch threads, so geniex-bench's NPU prefill throughput lagged llama-bench's by a wide margin on multi-core hosts. Mirror that default instead. Raising the thread count also exposed a latent bounds bug in the CPU pinning logic: reserved_cores=2 assumed n_threads always left slack below the core count, which no longer holds once n_threads can reach hardware_concurrency(). Scale reserved_cores down instead of pinning past the real core count or oversubscribing it. Signed-off-by: Mengsheng Wu --- sdk/plugins/llama_cpp/include/params.h | 11 +++++----- sdk/plugins/llama_cpp/src/params.cpp | 30 +++++++++++--------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/sdk/plugins/llama_cpp/include/params.h b/sdk/plugins/llama_cpp/include/params.h index 4934aeac0..6506a74a3 100644 --- a/sdk/plugins/llama_cpp/include/params.h +++ b/sdk/plugins/llama_cpp/include/params.h @@ -33,11 +33,12 @@ Device classify_device(const char* device_id, int n_gpu_layers); // Map a caller's config to llama params, filling each unset (0) field from the // plugin defaults. build_context_params resolves n_ctx (default differs between -// LLM at 4096 and VLM at 16384) plus n_batch / n_ubatch / n_seq_max, and picks -// thread counts via the per-(platform, device) rule (offloaded inference pins -// the upstream-fixed count; pure CPU uses cores/2). build_model_params only -// reads n_gpu_layers. Device selection and tensor-buffer overrides stay at the -// call site. +// LLM at 4096 and VLM at 16384) plus n_batch / n_ubatch / n_seq_max, and defaults +// n_threads / n_threads_batch to common_cpu_get_num_math() regardless of device — +// matching upstream llama-bench/llama-cli, which never vary thread count by +// backend. Callers may still override via ModelConfig's n_threads/n_threads_batch. +// build_model_params only reads n_gpu_layers. Device selection and tensor-buffer +// overrides stay at the call site. // // spec is the parsed speculative config (nullptr when disabled, and always // nullptr for the draft context itself): a target context that drafts needs diff --git a/sdk/plugins/llama_cpp/src/params.cpp b/sdk/plugins/llama_cpp/src/params.cpp index 12e61a324..278493740 100644 --- a/sdk/plugins/llama_cpp/src/params.cpp +++ b/sdk/plugins/llama_cpp/src/params.cpp @@ -10,6 +10,7 @@ #include #include +#include "common.h" #include "logging.h" #include "speculative.h" @@ -39,18 +40,7 @@ Device classify_device(const char* device_id, int n_gpu_layers) { return Device::CPU; // UNKOWN treat as CPU } -int resolve_n_threads(int requested, Device device) { - static const int cpu_threads = static_cast(std::thread::hardware_concurrency()); - static const int cpu_matrix[3][3] = { - {cpu_threads, 6, 6}, // Linux - {cpu_threads, 6, 6}, // Windows - {6, 6, 6} // Android - }; - - if (requested > 0) return requested; - - return cpu_matrix[static_cast(kHostPlatform)][static_cast(device)]; -} +int resolve_n_threads(int requested) { return requested > 0 ? requested : common_cpu_get_num_math(); } llama_model_params build_model_params(const geniex_ModelConfig& config, Device device) { static const bool mmap_matrix[3][3] = { @@ -113,8 +103,8 @@ llama_context_params build_context_params( cpar.n_batch = config.n_batch > 0 ? config.n_batch : 2048; cpar.n_ubatch = config.n_ubatch > 0 ? config.n_ubatch : ubatch; cpar.n_seq_max = config.n_seq_max > 0 ? config.n_seq_max : 1; - cpar.n_threads = resolve_n_threads(config.n_threads, device); - cpar.n_threads_batch = resolve_n_threads(config.n_threads_batch, device); + cpar.n_threads = resolve_n_threads(config.n_threads); + cpar.n_threads_batch = resolve_n_threads(config.n_threads_batch); cpar.flash_attn_type = static_cast(fa); cpar.no_perf = false; @@ -161,13 +151,19 @@ ggml_threadpool_params build_threadpool_params(int n_threads, Device device) { {1000, 1000, 1000} // Android }; - int reserved_cores = 2; - uint32_t poll = poll_matrix[static_cast(kHostPlatform)][static_cast(device)]; - bool pin = pin_matrix[static_cast(kHostPlatform)][static_cast(device)]; + uint32_t poll = poll_matrix[static_cast(kHostPlatform)][static_cast(device)]; + bool pin = pin_matrix[static_cast(kHostPlatform)][static_cast(device)]; ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); if (pin) { + // Leave up to 2 cores free for the OS/other work, but only if n_threads leaves + // that slack; n_threads can now reach hardware_concurrency() (see + // resolve_n_threads), and reserving cores on top of an already-full thread + // count would either pin past the real core count or oversubscribe the + // remaining ones. + int hw_threads = static_cast(std::thread::hardware_concurrency()); + int reserved_cores = hw_threads > 0 ? std::max(0, std::min(2, hw_threads - n_threads)) : 2; for (int i = 0; i < n_threads; ++i) { tpp.cpumask[reserved_cores + i] = true; } From dc31784da4a270e05b04100cae7da22427463269 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Wed, 26 Aug 2026 22:20:22 +0800 Subject: [PATCH 2/2] style(sdk): trim redundant reserved_cores comment The rationale is already covered by the commit message; keep the inline comment to one line. Signed-off-by: Mengsheng Wu --- sdk/plugins/llama_cpp/src/params.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sdk/plugins/llama_cpp/src/params.cpp b/sdk/plugins/llama_cpp/src/params.cpp index 278493740..9ee704498 100644 --- a/sdk/plugins/llama_cpp/src/params.cpp +++ b/sdk/plugins/llama_cpp/src/params.cpp @@ -157,11 +157,8 @@ ggml_threadpool_params build_threadpool_params(int n_threads, Device device) { ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); if (pin) { - // Leave up to 2 cores free for the OS/other work, but only if n_threads leaves - // that slack; n_threads can now reach hardware_concurrency() (see - // resolve_n_threads), and reserving cores on top of an already-full thread - // count would either pin past the real core count or oversubscribe the - // remaining ones. + // Reserve up to 2 cores only if n_threads leaves that slack, to avoid pinning past + // the real core count or oversubscribing it now that n_threads can reach hardware_concurrency(). int hw_threads = static_cast(std::thread::hardware_concurrency()); int reserved_cores = hw_threads > 0 ? std::max(0, std::min(2, hw_threads - n_threads)) : 2; for (int i = 0; i < n_threads; ++i) {