diff --git a/cli/server/handler/completion.go b/cli/server/handler/completion.go index 7bbbc3835..29287d52e 100644 --- a/cli/server/handler/completion.go +++ b/cli/server/handler/completion.go @@ -160,9 +160,17 @@ func Completions(c *gin.Context) { return } + // qairt rejects any generate call that carries stop sequences at all + // (sdk/plugins/qairt/src/llm.cpp:219: "--stop / --stop-file (stop + // sequences) is not supported by the qairt plugin"). Enforce them host + // side instead of forwarding them: withhold Stop from the plugin config + // and match the generated text against the requested stop strings as it + // comes in, via a stopMatcher. + stopSequences := completionStop(req.Stop) + hostStop := paths.RuntimeID == geniex_sdk.RuntimeQairt && len(stopSequences) > 0 + genConfig := &geniex_sdk.GenerationConfig{ MaxTokens: int32(req.MaxTokens.Value), - Stop: completionStop(req.Stop), SamplerConfig: &geniex_sdk.SamplerConfig{ Temperature: float32(req.Temperature.Value), TopP: float32(req.TopP.Value), @@ -174,6 +182,9 @@ func Completions(c *gin.Context) { Seed: int32(req.Seed.Value), }, } + if !hostStop { + genConfig.Stop = stopSequences + } echo := "" if req.Echo.Valid() && req.Echo.Value { echo = prompt @@ -191,6 +202,11 @@ func Completions(c *gin.Context) { wg sync.WaitGroup ) + var matcher *stopMatcher + if hostStop { + matcher = newStopMatcher(stopSequences) + } + wg.Add(1) go func() { defer wg.Done() @@ -200,8 +216,15 @@ func Completions(c *gin.Context) { if stopGen.Load() { return false } - dataCh <- token - return true + if matcher == nil { + dataCh <- token + return true + } + safe, stopped := matcher.feed(token) + if safe != "" { + dataCh <- safe + } + return !stopped }, Config: genConfig, }) @@ -220,10 +243,23 @@ func Completions(c *gin.Context) { } } else { // blocking - out, err := p.Generate(geniex_sdk.LlmGenerateInput{ + input := geniex_sdk.LlmGenerateInput{ PromptUTF8: prompt, Config: genConfig, - }) + } + var text string + if hostStop { + // No streaming consumer to feed here — the matcher's only job is + // to spot the stop sequence and cancel generation early via the + // OnToken return value, same as the streaming path. + matcher := newStopMatcher(stopSequences) + input.OnToken = func(token string) bool { + safe, stopped := matcher.feed(token) + text += safe + return !stopped + } + } + out, err := p.Generate(input) // A prompt that never fit is a client error (400). A window exhausted // mid-generation is a normal truncated completion (finish_reason=length), // so it falls through to the regular response below. @@ -235,7 +271,10 @@ func Completions(c *gin.Context) { c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error(), "code": geniex_sdk.SDKErrorCode(err)}) return } - writeCompletionResponse(c, echo+out.FullText, out.ProfileData) + if !hostStop { + text = out.FullText + } + writeCompletionResponse(c, echo+text, out.ProfileData) } } diff --git a/cli/server/handler/stop_match.go b/cli/server/handler/stop_match.go new file mode 100644 index 000000000..8d5521df2 --- /dev/null +++ b/cli/server/handler/stop_match.go @@ -0,0 +1,79 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package handler + +import "strings" + +// stopMatcher enforces OpenAI-style stop sequences host side, for plugins +// (qairt) that reject the parameter outright. It buffers just enough of the +// generated tail to catch a stop sequence split across two token +// boundaries, without holding back more text than necessary. +type stopMatcher struct { + stops []string + buf string +} + +func newStopMatcher(stops []string) *stopMatcher { + return &stopMatcher{stops: stops} +} + +// feed appends token to the internal buffer and returns the portion that is +// now safe to emit to the client, plus whether a stop sequence has fully +// matched. Once stopped is true, the caller must cancel generation and stop +// calling feed — the match itself (and anything after it) is dropped. +func (m *stopMatcher) feed(token string) (safe string, stopped bool) { + m.buf += token + + if idx := firstStopIndex(m.buf, m.stops); idx >= 0 { + safe = m.buf[:idx] + m.buf = "" + return safe, true + } + + // Hold back a suffix that could still grow into a stop sequence on the + // next token; only the rest is guaranteed clean of any stop sequence. + hold := longestStopPrefixSuffixLen(m.buf, m.stops) + safe, m.buf = m.buf[:len(m.buf)-hold], m.buf[len(m.buf)-hold:] + return safe, false +} + +// firstStopIndex returns the earliest byte offset in buf where a stop +// sequence starts, or -1 if none has fully matched yet. +func firstStopIndex(buf string, stops []string) int { + idx := -1 + for _, s := range stops { + if s == "" { + continue + } + if i := strings.Index(buf, s); i >= 0 && (idx == -1 || i < idx) { + idx = i + } + } + return idx +} + +// longestStopPrefixSuffixLen returns the length of the longest suffix of buf +// that is also a strict prefix of some stop sequence — the part that must be +// held back because the next token could still complete a match. +func longestStopPrefixSuffixLen(buf string, stops []string) int { + longest := 0 + for _, s := range stops { + if s == "" { + continue + } + limit := len(s) - 1 + if limit > len(buf) { + limit = len(buf) + } + for l := limit; l > 0; l-- { + if strings.HasSuffix(buf, s[:l]) { + if l > longest { + longest = l + } + break + } + } + } + return longest +} diff --git a/cli/server/handler/stop_match_test.go b/cli/server/handler/stop_match_test.go new file mode 100644 index 000000000..e8c35dbf2 --- /dev/null +++ b/cli/server/handler/stop_match_test.go @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries. +// SPDX-License-Identifier: BSD-3-Clause + +package handler + +import ( + "strings" + "testing" +) + +// feedAll drives a stopMatcher token by token and returns the concatenated +// safe text plus whether a stop sequence matched, stopping early (as a real +// caller would) once feed reports a match. +func feedAll(m *stopMatcher, tokens []string) (string, bool) { + var out strings.Builder + for _, tok := range tokens { + safe, stopped := m.feed(tok) + out.WriteString(safe) + if stopped { + return out.String(), true + } + } + return out.String(), false +} + +func TestStopMatcher(t *testing.T) { + tests := []struct { + name string + stops []string + tokens []string + wantText string + wantStop bool + }{ + {"no stops configured", nil, []string{"hello", " world"}, "hello world", false}, + {"no match", []string{"STOP"}, []string{"hello", " world"}, "hello world", false}, + {"match within a single token", []string{"STOP"}, []string{"abc", "defSTOPghi"}, "abcdef", true}, + { + // The FIM repro from #1341: an editor's stop list, split across + // streamed token boundaries the way real generation delivers it. + "stop sequence split across tokens", + []string{"", "", "", "<|endoftext|>"}, + []string{"def main():\n print(", "", ")\n"}, + "def main():\n print(", + true, + }, + {"earliest of several stops wins", []string{"", ""}, []string{"abcdefghi"}, "abc", true}, + {"empty stop string is ignored", []string{""}, []string{"hello"}, "hello", false}, + {"match at the very first token", []string{"STOP"}, []string{"STOPtrailing"}, "", true}, + {"partial match that never completes is still flushed", []string{""}, []string{"abc"}) + safe, stopped := m.feed("abc