From 9ce707160d089a3c79fbd0f5031b28eab9b3dd7e Mon Sep 17 00:00:00 2001 From: Shishir2405 Date: Thu, 20 Aug 2026 21:46:51 +0530 Subject: [PATCH] fix(cli): enforce /v1/completions stop sequences host side for qairt The qairt plugin rejects any generate call that carries stop sequences (sdk/plugins/qairt/src/llm.cpp:219), so completion.go forwarding Stop unchanged to every runtime made /v1/completions unusable with QAIRT models for any client that sends a stop list, which real FIM autocompletion clients always do. Withhold Stop from the plugin config when the runtime is qairt and match the requested stop strings against the generated text instead, via a new stopMatcher that buffers just enough of the tail to catch a match split across token boundaries. A match cancels generation early through the existing OnToken return-false path and truncates the returned text at the match, for both the streaming and blocking response paths. Closes #1341 Signed-off-by: Shishir2405 --- cli/server/handler/completion.go | 51 +++++++++++++++-- cli/server/handler/stop_match.go | 79 +++++++++++++++++++++++++++ cli/server/handler/stop_match_test.go | 69 +++++++++++++++++++++++ 3 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 cli/server/handler/stop_match.go create mode 100644 cli/server/handler/stop_match_test.go diff --git a/cli/server/handler/completion.go b/cli/server/handler/completion.go index 7bbbc3835..29287d52e 100644 --- a/cli/server/handler/completion.go +++ b/cli/server/handler/completion.go @@ -160,9 +160,17 @@ func Completions(c *gin.Context) { return } + // qairt rejects any generate call that carries stop sequences at all + // (sdk/plugins/qairt/src/llm.cpp:219: "--stop / --stop-file (stop + // sequences) is not supported by the qairt plugin"). Enforce them host + // side instead of forwarding them: withhold Stop from the plugin config + // and match the generated text against the requested stop strings as it + // comes in, via a stopMatcher. + stopSequences := completionStop(req.Stop) + hostStop := paths.RuntimeID == geniex_sdk.RuntimeQairt && len(stopSequences) > 0 + genConfig := &geniex_sdk.GenerationConfig{ MaxTokens: int32(req.MaxTokens.Value), - Stop: completionStop(req.Stop), SamplerConfig: &geniex_sdk.SamplerConfig{ Temperature: float32(req.Temperature.Value), TopP: float32(req.TopP.Value), @@ -174,6 +182,9 @@ func Completions(c *gin.Context) { Seed: int32(req.Seed.Value), }, } + if !hostStop { + genConfig.Stop = stopSequences + } echo := "" if req.Echo.Valid() && req.Echo.Value { echo = prompt @@ -191,6 +202,11 @@ func Completions(c *gin.Context) { wg sync.WaitGroup ) + var matcher *stopMatcher + if hostStop { + matcher = newStopMatcher(stopSequences) + } + wg.Add(1) go func() { defer wg.Done() @@ -200,8 +216,15 @@ func Completions(c *gin.Context) { if stopGen.Load() { return false } - dataCh <- token - return true + if matcher == nil { + dataCh <- token + return true + } + safe, stopped := matcher.feed(token) + if safe != "" { + dataCh <- safe + } + return !stopped }, Config: genConfig, }) @@ -220,10 +243,23 @@ func Completions(c *gin.Context) { } } else { // blocking - out, err := p.Generate(geniex_sdk.LlmGenerateInput{ + input := geniex_sdk.LlmGenerateInput{ PromptUTF8: prompt, Config: genConfig, - }) + } + var text string + if hostStop { + // No streaming consumer to feed here — the matcher's only job is + // to spot the stop sequence and cancel generation early via the + // OnToken return value, same as the streaming path. + matcher := newStopMatcher(stopSequences) + input.OnToken = func(token string) bool { + safe, stopped := matcher.feed(token) + text += safe + return !stopped + } + } + out, err := p.Generate(input) // A prompt that never fit is a client error (400). A window exhausted // mid-generation is a normal truncated completion (finish_reason=length), // so it falls through to the regular response below. @@ -235,7 +271,10 @@ func Completions(c *gin.Context) { c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) return } - writeCompletionResponse(c, echo+out.FullText, out.ProfileData) + if !hostStop { + text = out.FullText + } + writeCompletionResponse(c, echo+text, out.ProfileData) } } diff --git a/cli/server/handler/stop_match.go b/cli/server/handler/stop_match.go new file mode 100644 index 000000000..8d5521df2 --- /dev/null +++ b/cli/server/handler/stop_match.go @@ -0,0 +1,79 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package handler + +import "strings" + +// stopMatcher enforces OpenAI-style stop sequences host side, for plugins +// (qairt) that reject the parameter outright. It buffers just enough of the +// generated tail to catch a stop sequence split across two token +// boundaries, without holding back more text than necessary. +type stopMatcher struct { + stops []string + buf string +} + +func newStopMatcher(stops []string) *stopMatcher { + return &stopMatcher{stops: stops} +} + +// feed appends token to the internal buffer and returns the portion that is +// now safe to emit to the client, plus whether a stop sequence has fully +// matched. Once stopped is true, the caller must cancel generation and stop +// calling feed — the match itself (and anything after it) is dropped. +func (m *stopMatcher) feed(token string) (safe string, stopped bool) { + m.buf += token + + if idx := firstStopIndex(m.buf, m.stops); idx >= 0 { + safe = m.buf[:idx] + m.buf = "" + return safe, true + } + + // Hold back a suffix that could still grow into a stop sequence on the + // next token; only the rest is guaranteed clean of any stop sequence. + hold := longestStopPrefixSuffixLen(m.buf, m.stops) + safe, m.buf = m.buf[:len(m.buf)-hold], m.buf[len(m.buf)-hold:] + return safe, false +} + +// firstStopIndex returns the earliest byte offset in buf where a stop +// sequence starts, or -1 if none has fully matched yet. +func firstStopIndex(buf string, stops []string) int { + idx := -1 + for _, s := range stops { + if s == "" { + continue + } + if i := strings.Index(buf, s); i >= 0 && (idx == -1 || i < idx) { + idx = i + } + } + return idx +} + +// longestStopPrefixSuffixLen returns the length of the longest suffix of buf +// that is also a strict prefix of some stop sequence — the part that must be +// held back because the next token could still complete a match. +func longestStopPrefixSuffixLen(buf string, stops []string) int { + longest := 0 + for _, s := range stops { + if s == "" { + continue + } + limit := len(s) - 1 + if limit > len(buf) { + limit = len(buf) + } + for l := limit; l > 0; l-- { + if strings.HasSuffix(buf, s[:l]) { + if l > longest { + longest = l + } + break + } + } + } + return longest +} diff --git a/cli/server/handler/stop_match_test.go b/cli/server/handler/stop_match_test.go new file mode 100644 index 000000000..e8c35dbf2 --- /dev/null +++ b/cli/server/handler/stop_match_test.go @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package handler + +import ( + "strings" + "testing" +) + +// feedAll drives a stopMatcher token by token and returns the concatenated +// safe text plus whether a stop sequence matched, stopping early (as a real +// caller would) once feed reports a match. +func feedAll(m *stopMatcher, tokens []string) (string, bool) { + var out strings.Builder + for _, tok := range tokens { + safe, stopped := m.feed(tok) + out.WriteString(safe) + if stopped { + return out.String(), true + } + } + return out.String(), false +} + +func TestStopMatcher(t *testing.T) { + tests := []struct { + name string + stops []string + tokens []string + wantText string + wantStop bool + }{ + {"no stops configured", nil, []string{"hello", " world"}, "hello world", false}, + {"no match", []string{"STOP"}, []string{"hello", " world"}, "hello world", false}, + {"match within a single token", []string{"STOP"}, []string{"abc", "defSTOPghi"}, "abcdef", true}, + { + // The FIM repro from #1341: an editor's stop list, split across + // streamed token boundaries the way real generation delivers it. + "stop sequence split across tokens", + []string{"", "", "", "<|endoftext|>"}, + []string{"def main():\n print(", "", ")\n"}, + "def main():\n print(", + true, + }, + {"earliest of several stops wins", []string{"", ""}, []string{"abcdefghi"}, "abc", true}, + {"empty stop string is ignored", []string{""}, []string{"hello"}, "hello", false}, + {"match at the very first token", []string{"STOP"}, []string{"STOPtrailing"}, "", true}, + {"partial match that never completes is still flushed", []string{""}, []string{"abc"}) + safe, stopped := m.feed("abc