diff --git a/cli/cmd/geniex/serve.go b/cli/cmd/geniex/serve.go index 4c93c0467..323358f1c 100644 --- a/cli/cmd/geniex/serve.go +++ b/cli/cmd/geniex/serve.go @@ -30,6 +30,8 @@ func serve() *cobra.Command { serveCmd.Flags().String("host", "127.0.0.1:18181", "Default server address (env: GENIEX_HOST)") serveCmd.Flags().String("origins", "*", "Default CORS origins (env: GENIEX_ORIGINS)") serveCmd.Flags().Int("keepalive", 300, "Keepalive seconds (env: GENIEX_KEEPALIVE)") + serveCmd.Flags().String("webui-dir", "", "Serve a llama.cpp Web UI static build from this directory (env: GENIEX_WEBUIDIR)") + serveCmd.Flags().String("model", "", "Default model for requests that omit model (env: GENIEX_MODEL)") // Model-load defaults applied when a request omits them (llama_cpp only; // per-request body fields still override). serveCmd.Flags().Int32("nctx", 4096, "Default context window size, llama_cpp only (env: GENIEX_NCTX)") @@ -43,6 +45,8 @@ func serve() *cobra.Command { viper.BindPFlag("host", serveCmd.Flags().Lookup("host")) viper.BindPFlag("origins", serveCmd.Flags().Lookup("origins")) viper.BindPFlag("keepalive", serveCmd.Flags().Lookup("keepalive")) + viper.BindPFlag("webuidir", serveCmd.Flags().Lookup("webui-dir")) + viper.BindPFlag("model", serveCmd.Flags().Lookup("model")) viper.BindPFlag("nctx", serveCmd.Flags().Lookup("nctx")) viper.BindPFlag("ngl", serveCmd.Flags().Lookup("ngl")) viper.BindPFlag("compute", serveCmd.Flags().Lookup("compute")) diff --git a/cli/internal/config/config.go b/cli/internal/config/config.go index 3c9fafe69..f266f53f1 100644 --- a/cli/internal/config/config.go +++ b/cli/internal/config/config.go @@ -19,6 +19,8 @@ type Config struct { Host string // Server host and port (default: "127.0.0.1:18181") Origins string // Allowed CORS origins (default: "*") KeepAlive int64 // Connection keep-alive timeout in seconds (default: 300) + WebUIDir string // Optional llama.cpp Web UI static build directory + Model string // Default model for requests that omit the model field // Model-load defaults applied when a request omits them (llama_cpp only; // per-request body fields still override). Compute is the alias resolved by // the SDK (sdk/src/device.cpp); empty means the SDK's own default. @@ -40,6 +42,8 @@ func init() { // ENV only param need to set default here viper.SetDefault("hftoken", "") // Default empty token viper.SetDefault("log", "none") // Default log level + viper.SetDefault("webuidir", "") + viper.SetDefault("model", "") viper.SetEnvPrefix("geniex") viper.AutomaticEnv() diff --git a/cli/server/BUILD.bazel b/cli/server/BUILD.bazel index 1e5fbf425..03fa7002b 100644 --- a/cli/server/BUILD.bazel +++ b/cli/server/BUILD.bazel @@ -6,6 +6,7 @@ go_library( srcs = [ "route.go", "server.go", + "webui.go", ], importpath = "github.com/qualcomm/GenieX/cli/server", visibility = ["//visibility:public"], @@ -26,4 +27,5 @@ go_cgo_test( "server_test.go", ], embed = [":server"], + deps = ["@com_github_gin_gonic_gin//:gin"], ) diff --git a/cli/server/handler/BUILD.bazel b/cli/server/handler/BUILD.bazel index 84fededd5..ce370fc01 100644 --- a/cli/server/handler/BUILD.bazel +++ b/cli/server/handler/BUILD.bazel @@ -29,4 +29,5 @@ go_cgo_test( name = "handler_test", srcs = ["package_test.go"], embed = [":handler"], + deps = ["//bindings/go:geniex_sdk_go"], ) diff --git a/cli/server/handler/chat.go b/cli/server/handler/chat.go index 37b986558..455c8d350 100644 --- a/cli/server/handler/chat.go +++ b/cli/server/handler/chat.go @@ -36,10 +36,16 @@ type ChatCompletionRequest struct { ChatCompletionNewParams Stream bool `json:"stream"` - EnableThink bool `json:"enable_think"` - NCtx int32 `json:"nctx"` - Ngl int32 `json:"ngl"` // 0 = pure CPU, -1 = all layers, N = N layers; defaults to the server --ngl when omitted - Compute string `json:"compute"` + EnableThink bool `json:"enable_think"` + // llama.cpp Web UI compatibility aliases. The UI uses llama-server's + // repeat_penalty name and puts enable_thinking under chat_template_kwargs. + RepeatPenalty *float32 `json:"repeat_penalty"` + ChatTemplateKwargs *struct { + EnableThinking *bool `json:"enable_thinking"` + } `json:"chat_template_kwargs"` + NCtx int32 `json:"nctx"` + Ngl int32 `json:"ngl"` // 0 = pure CPU, -1 = all layers, N = N layers; defaults to the server --ngl when omitted + Compute string `json:"compute"` ImageMaxLength int32 `json:"image_max_length"` @@ -66,6 +72,7 @@ func defaultChatCompletionRequest() ChatCompletionRequest { return ChatCompletionRequest{ ChatCompletionNewParams: ChatCompletionNewParams{ MaxCompletionTokens: param.NewOpt[int64](2048), + Model: openai.ChatModel(cfg.Model), }, Stream: false, @@ -83,6 +90,18 @@ func defaultChatCompletionRequest() ChatCompletionRequest { } } +func normalizeLlamaWebUIRequest(param *ChatCompletionRequest) { + if param.MaxTokens.Valid() { + param.MaxCompletionTokens = param.MaxTokens + } + if param.RepeatPenalty != nil { + param.RepetitionPenalty = *param.RepeatPenalty + } + if param.ChatTemplateKwargs != nil && param.ChatTemplateKwargs.EnableThinking != nil { + param.EnableThink = *param.ChatTemplateKwargs.EnableThinking + } +} + func isWarmupRequest(param ChatCompletionRequest) bool { if len(param.Messages) == 0 { return true @@ -103,6 +122,13 @@ func ChatCompletions(c *gin.Context) { c.JSON(http.StatusBadRequest, map[string]any{"error": err.Error()}) return } + normalizeLlamaWebUIRequest(¶m) + if param.Model == "" { + c.JSON(http.StatusBadRequest, map[string]any{ + "error": "model is required; provide it in the request or configure geniex serve --model", + }) + return + } slog.Info("ChatCompletions", "param", param) name, _ := geniex_sdk.SplitNamePrecision(param.Model) @@ -243,11 +269,13 @@ func chatCompletionsLLM(c *gin.Context, param ChatCompletionRequest, modelParam samplerConfig := parseSamplerConfig(param) - p, err := service.KeepAliveGet[geniex_sdk.LLM]( - string(param.Model), - modelParam, - c.GetHeader("GenieX-KeepCache") != "true", - ) + p, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.LLM, error) { + return service.KeepAliveGet[geniex_sdk.LLM]( + string(param.Model), + modelParam, + c.GetHeader("GenieX-KeepCache") != "true", + ) + }) if writeKeepAliveError(c, err) { return } @@ -256,11 +284,13 @@ func chatCompletionsLLM(c *gin.Context, param ChatCompletionRequest, modelParam return } - formatted, err := p.ApplyChatTemplate(geniex_sdk.LlmApplyChatTemplateInput{ - Messages: messages, - Tools: tools, - EnableThink: param.EnableThink, - AddGenerationPrompt: true, + formatted, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.LlmApplyChatTemplateOutput, error) { + return p.ApplyChatTemplate(geniex_sdk.LlmApplyChatTemplateInput{ + Messages: messages, + Tools: tools, + EnableThink: param.EnableThink, + AddGenerationPrompt: true, + }) }) if err != nil { c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) @@ -280,31 +310,34 @@ func chatCompletionsLLM(c *gin.Context, param ChatCompletionRequest, modelParam resWg.Add(1) go func() { defer resWg.Done() - res, err = p.Generate(geniex_sdk.LlmGenerateInput{ - PromptUTF8: formatted.FormattedText, - OnToken: func(token string) bool { - if stopGen { - return false - } - dataCh <- token - return true - }, - Config: &geniex_sdk.GenerationConfig{ - MaxTokens: int32(param.MaxCompletionTokens.Value), - SamplerConfig: samplerConfig, - }, + res, err = service.RunOnInferenceThreadResult(func() (*geniex_sdk.LlmGenerateOutput, error) { + return p.Generate(geniex_sdk.LlmGenerateInput{ + PromptUTF8: formatted.FormattedText, + OnToken: func(token string) bool { + if stopGen { + return false + } + dataCh <- token + return true + }, + Config: &geniex_sdk.GenerationConfig{ + MaxTokens: int32(param.MaxCompletionTokens.Value), + SamplerConfig: samplerConfig, + }, + }) }) close(dataCh) }() wait := func() error { resWg.Wait(); return err } usage := func() openai.CompletionUsage { return profile2Usage(res.ProfileData) } + timings := func() llamaTimings { return profile2Timings(res.ProfileData) } finish := func() string { return mapFinishReason(res.ProfileData.StopReason) } includeUsage := param.StreamOptions.IncludeUsage.Value if !parseTool { - streamPlainText(c, dataCh, wait, includeUsage, usage, finish) + streamPlainText(c, dataCh, wait, includeUsage, usage, timings, string(param.Model), finish) } else { - streamToolCall(c, dataCh, wait, includeUsage, usage, finish) + streamToolCall(c, dataCh, wait, includeUsage, usage, timings, string(param.Model), finish) } stopGen = true @@ -312,12 +345,14 @@ func chatCompletionsLLM(c *gin.Context, param ChatCompletionRequest, modelParam } } else { - genOut, err := p.Generate(geniex_sdk.LlmGenerateInput{ - PromptUTF8: formatted.FormattedText, - Config: &geniex_sdk.GenerationConfig{ - MaxTokens: int32(param.MaxCompletionTokens.Value), - SamplerConfig: samplerConfig, - }, + genOut, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.LlmGenerateOutput, error) { + return p.Generate(geniex_sdk.LlmGenerateInput{ + PromptUTF8: formatted.FormattedText, + Config: &geniex_sdk.GenerationConfig{ + MaxTokens: int32(param.MaxCompletionTokens.Value), + SamplerConfig: samplerConfig, + }, + }) }) if errors.Is(err, geniex_sdk.ErrLlmTokenizationContextLength) { writeContextLengthExceeded(c, genOut.FullText, genOut.ProfileData) @@ -327,7 +362,7 @@ func chatCompletionsLLM(c *gin.Context, param ChatCompletionRequest, modelParam c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) return } - writeBlockingResponse(c, genOut.FullText, genOut.ProfileData, parseTool) + writeBlockingResponse(c, genOut.FullText, genOut.ProfileData, parseTool, string(param.Model)) } } @@ -465,11 +500,13 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam samplerConfig := parseSamplerConfig(param) - p, err := service.KeepAliveGet[geniex_sdk.VLM]( - string(param.Model), - modelParam, - c.GetHeader("GenieX-KeepCache") != "true", - ) + p, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.VLM, error) { + return service.KeepAliveGet[geniex_sdk.VLM]( + string(param.Model), + modelParam, + c.GetHeader("GenieX-KeepCache") != "true", + ) + }) if writeKeepAliveError(c, err) { return } @@ -479,10 +516,12 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam } // Format prompt using VLM chat template - formatted, err := p.ApplyChatTemplate(geniex_sdk.VlmApplyChatTemplateInput{ - Messages: messages, - Tools: tools, - EnableThink: param.EnableThink, + formatted, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.VlmApplyChatTemplateOutput, error) { + return p.ApplyChatTemplate(geniex_sdk.VlmApplyChatTemplateInput{ + Messages: messages, + Tools: tools, + EnableThink: param.EnableThink, + }) }) if err != nil { c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) @@ -512,22 +551,24 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam resWg.Add(1) go func() { defer resWg.Done() - res, err = p.Generate(geniex_sdk.VlmGenerateInput{ - PromptUTF8: formatted.FormattedText, - OnToken: func(token string) bool { - if stopGen { - return false - } - dataCh <- token - return true - }, - Config: &geniex_sdk.GenerationConfig{ - MaxTokens: int32(param.MaxCompletionTokens.Value), - SamplerConfig: samplerConfig, - ImagePaths: images, - AudioPaths: audios, - ImageMaxLength: param.ImageMaxLength, - }, + res, err = service.RunOnInferenceThreadResult(func() (*geniex_sdk.VlmGenerateOutput, error) { + return p.Generate(geniex_sdk.VlmGenerateInput{ + PromptUTF8: formatted.FormattedText, + OnToken: func(token string) bool { + if stopGen { + return false + } + dataCh <- token + return true + }, + Config: &geniex_sdk.GenerationConfig{ + MaxTokens: int32(param.MaxCompletionTokens.Value), + SamplerConfig: samplerConfig, + ImagePaths: images, + AudioPaths: audios, + ImageMaxLength: param.ImageMaxLength, + }, + }) }) close(dataCh) @@ -535,12 +576,13 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam wait := func() error { resWg.Wait(); return err } usage := func() openai.CompletionUsage { return profile2Usage(res.ProfileData) } + timings := func() llamaTimings { return profile2Timings(res.ProfileData) } finish := func() string { return mapFinishReason(res.ProfileData.StopReason) } includeUsage := param.StreamOptions.IncludeUsage.Value if !parseTool { - streamPlainText(c, dataCh, wait, includeUsage, usage, finish) + streamPlainText(c, dataCh, wait, includeUsage, usage, timings, string(param.Model), finish) } else { - streamToolCall(c, dataCh, wait, includeUsage, usage, finish) + streamToolCall(c, dataCh, wait, includeUsage, usage, timings, string(param.Model), finish) } stopGen = true @@ -548,15 +590,17 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam } } else { - genOut, err := p.Generate(geniex_sdk.VlmGenerateInput{ - PromptUTF8: formatted.FormattedText, - Config: &geniex_sdk.GenerationConfig{ - MaxTokens: int32(param.MaxCompletionTokens.Value), - SamplerConfig: samplerConfig, - ImagePaths: images, - AudioPaths: audios, - ImageMaxLength: param.ImageMaxLength, - }, + genOut, err := service.RunOnInferenceThreadResult(func() (*geniex_sdk.VlmGenerateOutput, error) { + return p.Generate(geniex_sdk.VlmGenerateInput{ + PromptUTF8: formatted.FormattedText, + Config: &geniex_sdk.GenerationConfig{ + MaxTokens: int32(param.MaxCompletionTokens.Value), + SamplerConfig: samplerConfig, + ImagePaths: images, + AudioPaths: audios, + ImageMaxLength: param.ImageMaxLength, + }, + }) }) if errors.Is(err, geniex_sdk.ErrLlmTokenizationContextLength) && genOut != nil { writeContextLengthExceeded(c, genOut.FullText, genOut.ProfileData) @@ -566,7 +610,7 @@ func chatCompletionsVLM(c *gin.Context, param ChatCompletionRequest, modelParam c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) return } - writeBlockingResponse(c, genOut.FullText, genOut.ProfileData, parseTool) + writeBlockingResponse(c, genOut.FullText, genOut.ProfileData, parseTool, string(param.Model)) } } @@ -636,12 +680,13 @@ func writeContextLengthExceeded(c *gin.Context, fullText string, profile geniex_ }, "choices": []openai.ChatCompletionChoice{choice}, "usage": profile2Usage(profile), + "timings": profile2Timings(profile), }) } // writeBlockingResponse emits a non-streaming completion: tool-call response // when parseTool matches, content response otherwise (or on parse failure). -func writeBlockingResponse(c *gin.Context, fullText string, profile geniex_sdk.ProfileData, parseTool bool) { +func writeBlockingResponse(c *gin.Context, fullText string, profile geniex_sdk.ProfileData, parseTool bool, model string) { if parseTool { toolCall, err := utils.ParseToolCalls(fullText) if err == nil { @@ -653,22 +698,36 @@ func writeBlockingResponse(c *gin.Context, fullText string, profile geniex_sdk.P Type: "function", Function: toolCall, }} - c.JSON(http.StatusOK, openai.ChatCompletion{ - Choices: []openai.ChatCompletionChoice{choice}, - Usage: profile2Usage(profile), + c.JSON(http.StatusOK, map[string]any{ + "object": "chat.completion", + "model": model, + "choices": []openai.ChatCompletionChoice{choice}, + "usage": profile2Usage(profile), + "timings": profile2Timings(profile), }) return } slog.Warn("Tool call parse error, fallback to text", "error", err) } - choice := openai.ChatCompletionChoice{} - choice.FinishReason = mapFinishReason(profile.StopReason) - choice.Message.Role = constant.Assistant(openai.MessageRoleAssistant) - choice.Message.Content = fullText - c.JSON(http.StatusOK, openai.ChatCompletion{ - Choices: []openai.ChatCompletionChoice{choice}, - Usage: profile2Usage(profile), + reasoning, content := splitGemma4Response(fullText) + message := map[string]any{ + "role": openai.MessageRoleAssistant, + "content": content, + } + if reasoning != "" { + message["reasoning_content"] = reasoning + } + c.JSON(http.StatusOK, map[string]any{ + "object": "chat.completion", + "model": model, + "choices": []map[string]any{{ + "index": 0, + "finish_reason": mapFinishReason(profile.StopReason), + "message": message, + }}, + "usage": profile2Usage(profile), + "timings": profile2Timings(profile), }) } @@ -680,6 +739,9 @@ type streamUsage func() openai.CompletionUsage // stop_reason to the OpenAI finish_reason vocabulary for the final chunk. type streamFinish func() string +// streamTimings exposes the completed SDK profile after generation finishes. +type streamTimings func() llamaTimings + // The openai-go response structs marshal FinishReason as a plain string, so // every chunk carried `"finish_reason": ""` and no terminal chunk was sent. // The OpenAI streaming spec requires null on intermediate chunks and @@ -688,9 +750,10 @@ type streamFinish func() string // give the stream spec-compliant serialization. type streamDelta struct { - Role string `json:"role,omitempty"` - Content string `json:"content,omitempty"` - ToolCalls []openai.ChatCompletionChunkChoiceDeltaToolCall `json:"tool_calls,omitempty"` + Role string `json:"role,omitempty"` + Content string `json:"content,omitempty"` + ReasoningContent string `json:"reasoning_content,omitempty"` + ToolCalls []openai.ChatCompletionChunkChoiceDeltaToolCall `json:"tool_calls,omitempty"` } type streamChoice struct { @@ -701,8 +764,10 @@ type streamChoice struct { type streamChunk struct { Object string `json:"object"` + Model string `json:"model,omitempty"` Choices []streamChoice `json:"choices"` Usage *openai.CompletionUsage `json:"usage,omitempty"` + Timings *llamaTimings `json:"timings,omitempty"` } const streamChunkObject = "chat.completion.chunk" @@ -716,11 +781,116 @@ func contentChunk(content string) streamChunk { } } +func reasoningChunk(content string) streamChunk { + return streamChunk{ + Object: streamChunkObject, + Choices: []streamChoice{{ + Delta: streamDelta{Role: string(openai.MessageRoleAssistant), ReasoningContent: content}, + }}, + } +} + +const ( + gemma4ReasoningStart = "<|channel>thought" + gemma4ReasoningEnd = "" +) + +func splitGemma4Response(fullText string) (reasoning string, content string) { + start := strings.Index(fullText, gemma4ReasoningStart) + if start < 0 { + return "", fullText + } + reasoningStart := start + len(gemma4ReasoningStart) + endRelative := strings.Index(fullText[reasoningStart:], gemma4ReasoningEnd) + if endRelative < 0 { + return "", fullText + } + end := reasoningStart + endRelative + reasoning = strings.TrimSpace(fullText[reasoningStart:end]) + content = fullText[end+len(gemma4ReasoningEnd):] + return reasoning, content +} + +// gemma4StreamParser mirrors llama.cpp's Gemma 4 reasoning boundary parser +// for the control-token pieces emitted by the embedded llama_cpp plugin. +// Non-Gemma output is passed through unchanged. +type gemma4StreamParser struct { + inReasoning bool + trimReasoningLF bool + pendingChannel bool +} + +func (p *gemma4StreamParser) parse(piece string) (streamChunk, bool) { + if p.pendingChannel { + p.pendingChannel = false + if strings.HasPrefix(piece, "thought") { + p.inReasoning = true + p.trimReasoningLF = true + piece = strings.TrimPrefix(piece, "thought") + } else { + return contentChunk("<|channel>" + piece), true + } + } else if piece == "<|channel>" { + p.pendingChannel = true + return streamChunk{}, false + } + + if strings.HasPrefix(piece, gemma4ReasoningStart) { + p.inReasoning = true + p.trimReasoningLF = true + piece = strings.TrimPrefix(piece, gemma4ReasoningStart) + } + + if p.inReasoning { + if idx := strings.Index(piece, gemma4ReasoningEnd); idx >= 0 { + reasoning := piece[:idx] + p.inReasoning = false + p.trimReasoningLF = false + remainder := piece[idx+len(gemma4ReasoningEnd):] + if reasoning != "" && remainder != "" { + return streamChunk{ + Object: streamChunkObject, + Choices: []streamChoice{{Delta: streamDelta{ + Role: string(openai.MessageRoleAssistant), + ReasoningContent: reasoning, + Content: remainder, + }}}, + }, true + } + if reasoning != "" { + return reasoningChunk(reasoning), true + } + if remainder != "" { + return contentChunk(remainder), true + } + return streamChunk{}, false + } + if p.trimReasoningLF { + if piece == "" { + return streamChunk{}, false + } + piece = strings.TrimPrefix(piece, "\n") + p.trimReasoningLF = false + } + if piece == "" { + return streamChunk{}, false + } + return reasoningChunk(piece), true + } + + if piece == gemma4ReasoningEnd { + return streamChunk{}, false + } + return contentChunk(piece), true +} + // finishChunk is the terminal chunk: empty delta, non-null finish_reason. -func finishChunk(reason string) streamChunk { +func finishChunk(reason string, timings llamaTimings, model string) streamChunk { return streamChunk{ Object: streamChunkObject, + Model: model, Choices: []streamChoice{{FinishReason: &reason}}, + Timings: &timings, } } @@ -730,18 +900,21 @@ func usageChunk(u openai.CompletionUsage) streamChunk { // streamPlainText drains dataCh as content chunks, then emits the finishing // chunk, optional usage and [DONE]. -func streamPlainText(c *gin.Context, dataCh <-chan string, wait func() error, includeUsage bool, usage streamUsage, finish streamFinish) { +func streamPlainText(c *gin.Context, dataCh <-chan string, wait func() error, includeUsage bool, usage streamUsage, timings streamTimings, model string, finish streamFinish) { + parser := gemma4StreamParser{} c.Stream(func(w io.Writer) bool { r, ok := <-dataCh if ok { - c.SSEvent("", contentChunk(r)) + if chunk, emit := parser.parse(r); emit { + c.SSEvent("", chunk) + } return true } if err := wait(); err != nil { c.SSEvent("", map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) return false } - c.SSEvent("", finishChunk(finish())) + c.SSEvent("", finishChunk(finish(), timings(), model)) if includeUsage { c.SSEvent("", usageChunk(usage())) } @@ -753,7 +926,7 @@ func streamPlainText(c *gin.Context, dataCh <-chan string, wait func() error, in // streamToolCall buffers the stream and emits a tool-call chunk once dataCh // closes; falls back to a content chunk when parsing fails. Either way the // stream ends with a finishing chunk, optional usage and [DONE]. -func streamToolCall(c *gin.Context, dataCh <-chan string, wait func() error, includeUsage bool, usage streamUsage, finish streamFinish) { +func streamToolCall(c *gin.Context, dataCh <-chan string, wait func() error, includeUsage bool, usage streamUsage, timings streamTimings, model string, finish streamFinish) { buffer := strings.Builder{} c.Stream(func(w io.Writer) bool { r, ok := <-dataCh @@ -789,7 +962,7 @@ func streamToolCall(c *gin.Context, dataCh <-chan string, wait func() error, inc }}, }) } - c.SSEvent("", finishChunk(finishReason)) + c.SSEvent("", finishChunk(finishReason, timings(), model)) if includeUsage { c.SSEvent("", usageChunk(usage())) } @@ -808,6 +981,33 @@ func profile2Usage(p geniex_sdk.ProfileData) openai.CompletionUsage { } } +// llamaTimings is llama-server's timing extension consumed by the upstream +// Web UI. It is derived from the completed GenieX SDK profile, not wall-clock +// proxy timing, so the UI and geniex-bench use the same token accounting. +type llamaTimings struct { + PromptN int64 `json:"prompt_n"` + PromptMS float64 `json:"prompt_ms"` + PromptPerSecond float64 `json:"prompt_per_second"` + PredictedN int64 `json:"predicted_n"` + PredictedMS float64 `json:"predicted_ms"` + PredictedPerSecond float64 `json:"predicted_per_second"` + TTFTMS float64 `json:"ttft_ms"` + CacheN int64 `json:"cache_n"` +} + +func profile2Timings(p geniex_sdk.ProfileData) llamaTimings { + return llamaTimings{ + PromptN: p.PromptTokens, + PromptMS: float64(p.PromptTime) / 1000.0, + PromptPerSecond: p.PrefillSpeed, + PredictedN: p.GeneratedTokens, + PredictedMS: float64(p.DecodeTime) / 1000.0, + PredictedPerSecond: p.DecodingSpeed, + TTFTMS: float64(p.TTFT) / 1000.0, + CacheN: 0, + } +} + // mapFinishReason translates the SDK's stop_reason values into the OpenAI // finish_reason vocabulary. func mapFinishReason(stopReason string) string { diff --git a/cli/server/handler/package_test.go b/cli/server/handler/package_test.go index 185f7cceb..0956e4a54 100644 --- a/cli/server/handler/package_test.go +++ b/cli/server/handler/package_test.go @@ -3,6 +3,89 @@ package handler -import "testing" +import ( + "encoding/json" + "testing" + + geniex_sdk "github.com/qualcomm/GenieX/bindings/go" +) func TestPackageBuilds(t *testing.T) {} + +func TestProfile2TimingsUsesSDKMicroseconds(t *testing.T) { + got := profile2Timings(geniex_sdk.ProfileData{ + PromptTokens: 128, + GeneratedTokens: 32, + PromptTime: 2_500_000, + DecodeTime: 1_600_000, + TTFT: 2_550_000, + PrefillSpeed: 51.2, + DecodingSpeed: 20, + }) + if got.PromptN != 128 || got.PromptMS != 2500 || got.PromptPerSecond != 51.2 || + got.PredictedN != 32 || got.PredictedMS != 1600 || got.PredictedPerSecond != 20 || got.TTFTMS != 2550 { + t.Fatalf("unexpected timings: %+v", got) + } +} + +func TestFinishChunkCarriesAuthoritativeTimings(t *testing.T) { + timings := llamaTimings{PromptN: 10, PromptMS: 20, PredictedN: 30, PredictedMS: 40} + data, err := json.Marshal(finishChunk("stop", timings, "local/test:Q4_0")) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got["timings"] == nil { + t.Fatalf("final chunk omitted timings: %s", data) + } + if got["model"] != "local/test:Q4_0" { + t.Fatalf("final chunk omitted model: %s", data) + } +} + +func TestNormalizeLlamaWebUIRequest(t *testing.T) { + t.Setenv("GENIEX_MODEL", "local/gemma-4-E2B-it:Q4_0") + param := defaultChatCompletionRequest() + if err := json.Unmarshal([]byte(`{"max_tokens":64,"repeat_penalty":1.25,"chat_template_kwargs":{"enable_thinking":false}}`), ¶m); err != nil { + t.Fatal(err) + } + normalizeLlamaWebUIRequest(¶m) + if param.MaxCompletionTokens.Value != 64 { + t.Fatalf("max_tokens alias not applied: %d", param.MaxCompletionTokens.Value) + } + if param.RepetitionPenalty != 1.25 { + t.Fatalf("repeat_penalty alias not applied: %f", param.RepetitionPenalty) + } + if param.EnableThink { + t.Fatal("enable_thinking alias not applied") + } + if param.Model != "local/gemma-4-E2B-it:Q4_0" { + t.Fatalf("server default model not applied: %q", param.Model) + } +} + +func TestGemma4ReasoningParsingMatchesLlamaWebUIFields(t *testing.T) { + full := "<|channel>thought\ncheck the answerfinal answer" + reasoning, content := splitGemma4Response(full) + if reasoning != "check the answer" || content != "final answer" { + t.Fatalf("unexpected blocking parse: reasoning=%q content=%q", reasoning, content) + } + + parser := gemma4StreamParser{} + pieces := []string{"<|channel>", "thought", "\n", "check", " the answer", "", "final", " answer"} + var gotReasoning, gotContent string + for _, piece := range pieces { + chunk, emit := parser.parse(piece) + if !emit { + continue + } + gotReasoning += chunk.Choices[0].Delta.ReasoningContent + gotContent += chunk.Choices[0].Delta.Content + } + if gotReasoning != "check the answer" || gotContent != "final answer" { + t.Fatalf("unexpected stream parse: reasoning=%q content=%q", gotReasoning, gotContent) + } +} diff --git a/cli/server/route.go b/cli/server/route.go index f4e805591..159aeb3f4 100644 --- a/cli/server/route.go +++ b/cli/server/route.go @@ -4,20 +4,80 @@ package server import ( + "fmt" "net/http" + "os" + "path/filepath" + "strings" "github.com/gin-gonic/gin" + "github.com/qualcomm/GenieX/cli/internal/config" "github.com/qualcomm/GenieX/cli/server/docs" "github.com/qualcomm/GenieX/cli/server/handler" "github.com/qualcomm/GenieX/cli/server/middleware" ) -func RegisterRoot(r *gin.Engine) { +func RegisterRoot(r *gin.Engine) error { r.Use(middleware.CORS) + webuiDir := config.Get().WebUIDir + if webuiDir == "" { + r.GET("/", func(c *gin.Context) { + c.Redirect(http.StatusFound, "/docs/ui/") + }) + return nil + } + + root, err := filepath.Abs(webuiDir) + if err != nil { + return fmt.Errorf("resolve %q: %w", webuiDir, err) + } + index := filepath.Join(root, "index.html") + info, err := os.Stat(index) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("llama.cpp Web UI index not found: %s", index) + } + r.GET("/", func(c *gin.Context) { - c.Redirect(http.StatusFound, "/docs/ui/") + c.File(index) }) + r.GET("/props", WebUIProps) + r.HEAD("/props", WebUIProps) + r.GET("/health", func(c *gin.Context) { c.String(http.StatusOK, "ok") }) + r.GET("/slots", webUIDisabledFeature) + r.GET("/tools", webUIDisabledFeature) + r.POST("/tools", webUIDisabledFeature) + + // Preserve the upstream build exactly as generated. Unknown browser routes + // fall back to index.html for the Svelte SPA, while unknown API routes keep + // a real 404 instead of accidentally returning HTML. + r.NoRoute(func(c *gin.Context) { + if c.Request.Method != http.MethodGet && c.Request.Method != http.MethodHead { + c.Status(http.StatusNotFound) + return + } + path := c.Request.URL.Path + if strings.HasPrefix(path, "/v1/") || strings.HasPrefix(path, "/docs/") { + c.Status(http.StatusNotFound) + return + } + + rel := strings.TrimPrefix(filepath.Clean("/"+path), string(filepath.Separator)) + candidate := filepath.Join(root, rel) + withinRoot, relErr := filepath.Rel(root, candidate) + if relErr == nil && withinRoot != ".." && !strings.HasPrefix(withinRoot, ".."+string(filepath.Separator)) { + if asset, statErr := os.Stat(candidate); statErr == nil && asset.Mode().IsRegular() { + c.File(candidate) + return + } + } + c.File(index) + }) + return nil +} + +func webUIDisabledFeature(c *gin.Context) { + c.JSON(http.StatusForbidden, map[string]any{"error": "this feature is disabled"}) } // http://localhost:18181/docs/ui/ diff --git a/cli/server/server.go b/cli/server/server.go index 93a5bf2c5..c25127bcc 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -43,7 +43,10 @@ func Serve() { gin.SetMode(gin.ReleaseMode) engine := gin.Default() - RegisterRoot(engine) + if err := RegisterRoot(engine); err != nil { + fmt.Println(render.GetTheme().Error.Sprintf("Web UI configuration error: %v", err)) + return + } RegisterAPIv1(engine) RegisterSwagger(engine) diff --git a/cli/server/server_test.go b/cli/server/server_test.go index 396388a5e..2df861b75 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -4,8 +4,15 @@ package server import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" "strings" "testing" + + "github.com/gin-gonic/gin" ) func TestHostBindingHint(t *testing.T) { @@ -41,3 +48,60 @@ func TestHostBindingHint(t *testing.T) { }) } } + +func TestRegisterRootServesUnmodifiedWebUIAndCompatibilityProps(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "index.html"), []byte("upstream-ui"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "asset.js"), []byte("upstream-asset"), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("GENIEX_WEBUIDIR", dir) + t.Setenv("GENIEX_NCTX", "4096") + t.Setenv("GENIEX_MODEL", "local/test-model:Q4_0") + + gin.SetMode(gin.TestMode) + engine := gin.New() + if err := RegisterRoot(engine); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + path string + want string + }{ + {path: "/", want: "upstream-ui"}, + {path: "/asset.js", want: "upstream-asset"}, + {path: "/chat/example", want: "upstream-ui"}, + } { + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if recorder.Code != http.StatusOK || recorder.Body.String() != tc.want { + t.Fatalf("GET %s = (%d, %q), want (200, %q)", tc.path, recorder.Code, recorder.Body.String(), tc.want) + } + } + + propsRecorder := httptest.NewRecorder() + engine.ServeHTTP(propsRecorder, httptest.NewRequest(http.MethodGet, "/props", nil)) + var props map[string]any + if err := json.Unmarshal(propsRecorder.Body.Bytes(), &props); err != nil { + t.Fatal(err) + } + if propsRecorder.Code != http.StatusOK || props["role"] != "model" || props["model_alias"] != "local/test-model:Q4_0" { + t.Fatalf("unexpected /props response: %d %s", propsRecorder.Code, propsRecorder.Body.String()) + } + + missingAPI := httptest.NewRecorder() + engine.ServeHTTP(missingAPI, httptest.NewRequest(http.MethodGet, "/v1/missing", nil)) + if missingAPI.Code != http.StatusNotFound { + t.Fatalf("unknown API route returned %d, want 404", missingAPI.Code) + } +} + +func TestRegisterRootRejectsMissingWebUIBuild(t *testing.T) { + t.Setenv("GENIEX_WEBUIDIR", t.TempDir()) + if err := RegisterRoot(gin.New()); err == nil { + t.Fatal("expected missing index.html to fail") + } +} diff --git a/cli/server/service/BUILD.bazel b/cli/server/service/BUILD.bazel index b6cb0ae72..c07e92d71 100644 --- a/cli/server/service/BUILD.bazel +++ b/cli/server/service/BUILD.bazel @@ -4,6 +4,7 @@ load("//scripts:cgo_test.bzl", "go_cgo_test") go_library( name = "service", srcs = [ + "inference_thread.go", "keepalive.go", "service.go", ], diff --git a/cli/server/service/inference_thread.go b/cli/server/service/inference_thread.go new file mode 100644 index 000000000..3316e053d --- /dev/null +++ b/cli/server/service/inference_thread.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package service + +import "runtime" + +type inferenceThreadTask struct { + run func() + done chan struct{} +} + +var inferenceThreadTasks = make(chan inferenceThreadTask) + +func init() { + go func() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for task := range inferenceThreadTasks { + func() { + defer close(task.done) + task.run() + }() + } + }() +} + +// RunOnInferenceThread serializes SDK work on one persistent OS thread. The +// llama_cpp CPU backend uses an OpenMP worker team associated with its calling +// thread; allowing Go to move consecutive requests between OS threads can +// leave the first team parked and make later requests run near single-thread +// speed. +func RunOnInferenceThread(run func()) { + task := inferenceThreadTask{run: run, done: make(chan struct{})} + inferenceThreadTasks <- task + <-task.done +} + +// RunOnInferenceThreadResult is the result-returning form of +// RunOnInferenceThread. +func RunOnInferenceThreadResult[T any](run func() (T, error)) (result T, err error) { + RunOnInferenceThread(func() { + result, err = run() + }) + return result, err +} diff --git a/cli/server/webui.go b/cli/server/webui.go new file mode 100644 index 000000000..52ac28dbd --- /dev/null +++ b/cli/server/webui.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/qualcomm/GenieX/cli/internal/config" +) + +// WebUIProps implements the subset of llama-server's /props contract used by +// the unmodified llama.cpp Web UI. Values mirror GenieX request defaults so +// loading the UI does not silently change sampling behavior. +func WebUIProps(c *gin.Context) { + cfg := config.Get() + modelAlias := "GenieX" + if cfg.Model != "" { + modelAlias = cfg.Model + } + c.JSON(http.StatusOK, map[string]any{ + "role": "model", + "model_alias": modelAlias, + "model_path": "GenieX llama_cpp plugin", + "total_slots": 1, + "modalities": map[string]bool{"vision": false, "audio": false, "video": false}, + "chat_template": "", + "bos_token": "", + "eos_token": "", + "build_info": "GenieX llama_cpp Web UI compatibility", + "cors_proxy_enabled": false, + "default_generation_settings": map[string]any{ + "id": 0, + "id_task": -1, + "n_ctx": cfg.NCtx, + "speculative": false, + "is_processing": false, + "prompt": "", + "params": map[string]any{ + "n_predict": 2048, + "max_tokens": 2048, + "seed": 0, + "temperature": 0.0, + "dynatemp_range": 0.0, + "dynatemp_exponent": 1.0, + "top_k": 0, + "top_p": 0.0, + "min_p": 0.0, + "typical_p": 1.0, + "repeat_last_n": 0, + "repeat_penalty": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "dry_multiplier": 0.0, + "dry_base": 1.75, + "dry_allowed_length": 2, + "dry_penalty_last_n": 0, + "samplers": []string{}, + "backend_sampling": false, + "stream": true, + "timings_per_token": false, + }, + "next_token": map[string]any{ + "has_next_token": false, + "has_new_line": false, + "n_remain": 0, + "n_decoded": 0, + "stopping_word": "", + }, + }, + }) +} diff --git a/sdk/plugins/llama_cpp/src/params.cpp b/sdk/plugins/llama_cpp/src/params.cpp index b5b139c9c..d0ad30244 100644 --- a/sdk/plugins/llama_cpp/src/params.cpp +++ b/sdk/plugins/llama_cpp/src/params.cpp @@ -69,19 +69,24 @@ llama_model_params build_model_params(const geniex_ModelConfig& config, Device d } llama_context_params build_context_params(const geniex_ModelConfig& config, int32_t n_ctx_default, Device device) { + // Linux/Windows CPU defaults mirror llama-server's common_params. Keep the + // existing accelerator-specific values for GPU, NPU, and Android. static const uint32_t ubatch_matrix[3][3] = { - {2048, 512, 1024}, // Linux - {2048, 512, 1024}, // Windows + {512, 512, 1024}, // Linux + {512, 512, 1024}, // Windows {1024, 512, 1024} // Android }; - static const bool fa_matrix[3][3] = { - {true, false, true}, // Linux - {true, false, true}, // Windows - {true, false, true} // Android + static const llama_flash_attn_type fa_matrix[3][3] = { + // Linux + {LLAMA_FLASH_ATTN_TYPE_AUTO, LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED}, + // Windows + {LLAMA_FLASH_ATTN_TYPE_AUTO, LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED}, + // Android + {LLAMA_FLASH_ATTN_TYPE_ENABLED, LLAMA_FLASH_ATTN_TYPE_DISABLED, LLAMA_FLASH_ATTN_TYPE_ENABLED} }; - uint32_t ubatch = ubatch_matrix[static_cast(kHostPlatform)][static_cast(device)]; - bool fa = fa_matrix[static_cast(kHostPlatform)][static_cast(device)]; + uint32_t ubatch = ubatch_matrix[static_cast(kHostPlatform)][static_cast(device)]; + llama_flash_attn_type fa = fa_matrix[static_cast(kHostPlatform)][static_cast(device)]; llama_context_params cpar = llama_context_default_params(); cpar.n_ctx = config.n_ctx > 0 ? config.n_ctx : n_ctx_default; @@ -90,7 +95,11 @@ llama_context_params build_context_params(const geniex_ModelConfig& config, int3 cpar.n_seq_max = config.n_seq_max > 0 ? config.n_seq_max : 1; cpar.n_threads = resolve_n_threads(config.n_threads, device); cpar.n_threads_batch = resolve_n_threads(config.n_threads_batch, device); - cpar.flash_attn_type = static_cast(fa); + cpar.flash_attn_type = fa; + // Match llama-server's application default. llama_context_default_params() + // enables a full-size SWA cache, but common_params disables it so models + // such as Gemma 4 retain their intended sliding-window cache size. + cpar.swa_full = false; cpar.no_perf = false; GENIEX_LOG_INFO(