From db4ea90fa66b33f89fe4543e676ee0eaa1c9e2a2 Mon Sep 17 00:00:00 2001 From: hetaoBackend Date: Sun, 20 Sep 2026 11:08:54 +0800 Subject: [PATCH] fix: persist explicit BYOK image capabilities Add provider add --support-image and preserve capability metadata through create and update inputs. Cover saved configuration and actual image transport with the packaged offline BYOK fixture. Adapted the CLI declaration and capability preservation approach from smartworldarafath/minimax-code commit 6ec8ad084ea53dffc2786e6c9e931f2714b17306. Keep explicit image-input semantics without inferring vision from generic attachments or catalog fallbacks. Co-authored-by: smartworldarafath --- docs/examples.md | 35 +++++++++++++++ .../process-local-application-contract.ts | 25 ++++------- .../session/process-local-application.ts | 18 +++----- .../src/service/model-system/contracts.ts | 1 + .../management/service-input.test.ts | 25 +++++++++++ .../model-system/management/service-input.ts | 3 ++ .../model-system/resolution/model-ref.test.ts | 13 ++++++ packages/tui/src/cli/program.ts | 3 ++ packages/tui/src/cli/provider-command.ts | 2 + packages/tui/src/provider/contract.ts | 3 ++ test/byok.test.mjs | 45 +++++++++++++++++++ 11 files changed, 143 insertions(+), 30 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 12a536569..748cad8a4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -70,6 +70,41 @@ pnpm mcode provider list --json ## 3. Search and image input +For a custom BYOK model, declare image input support explicitly when adding the +provider. Use this only if the selected provider endpoint and model accept images: + +```bash +pnpm mcode provider add --name my-vision-provider --base-url https://example.com/v1 \ + --api-format openai-completions --model my-vision-model \ + --api-key-env MCODE_PROVIDER_API_KEY --support-image --use +``` + +`--support-image` applies to every repeated `--model` and saves +`capabilities.support_image: true`. Without it, adding an unknown custom model +does not infer image support from its name. `--use` and `provider test` check +connectivity with a text request; they do not verify vision support. + +For an existing provider, close MCode and add the capability to the matching model +in the active profile's `config.yaml` (normally `~/.minimax/config.yaml`; see +[Accounts and data](installation.md#accounts-and-data) for profiles and overrides). +Merge this fragment into the existing provider and keep its other settings: + +```yaml +custom_provider: + my-vision-provider: + models: + my-vision-model: + capabilities: + support_image: true +``` + +The existing `modalities: { input: [text, image], output: [text] }` model setting +also enables image input. Either declaration is sufficient. To make a model +text-only again, remove `image` from `modalities.input` and remove or set +`capabilities.support_image` to `false`. Generic `attachment: true` alone does not +declare image input support. Restart MCode after editing configuration and select +the configured model with `/provider`, or use `exec --model` for a single run. + After signing in to MiniMax, try a task that explicitly requires search: > Use web_search to find the official Node.js test runner documentation. Summarize how to run tests and include the source URL. If the tool is unavailable, say so. diff --git a/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts b/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts index f000d26ff..ff803b1ae 100644 --- a/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts +++ b/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts @@ -26,8 +26,13 @@ import type { CodexOAuthStartResult, CodexOAuthLoginOptions, CodexOAuthStatus, + UserModelInputView, } from "../../service/model-system/index.js"; +export type ProcessLocalModelInput = Omit & { + modalities?: { input?: readonly string[]; output?: readonly string[] }; +}; + export interface ProcessLocalMiniAppLaunch { readonly miniApp: MiniAppSurfaceSummary; /** Main-process-only launch coordinates. Never forward this object to a renderer. */ @@ -262,7 +267,7 @@ export interface LocalRuntimeApplication { baseUrl: string; apiKey: string; apiFormat?: string; - models?: readonly { modelId: string; displayName?: string }[]; + models?: readonly ProcessLocalModelInput[]; saveAndUse?: boolean; }): Promise; discoverCandidate(input: { @@ -278,21 +283,7 @@ export interface LocalRuntimeApplication { baseUrl: string; apiKey?: string; apiFormat?: string; - models?: readonly { - modelId: string; - displayName?: string; - configurationSource?: string; - enabled?: boolean; - attachment?: boolean; - reasoning?: boolean; - toolCall?: boolean; - temperature?: boolean; - modalities?: { - input?: readonly string[]; - output?: readonly string[]; - }; - limit?: { context?: number; output?: number }; - }[]; + models?: readonly ProcessLocalModelInput[]; }; modelId?: string; saveAndUse?: boolean; @@ -309,7 +300,7 @@ export interface LocalRuntimeApplication { apiKey?: string; apiFormat?: string; enabled?: boolean; - models?: readonly { modelId: string; displayName?: string }[]; + models?: readonly ProcessLocalModelInput[]; saveAndUse?: boolean; }): Promise; delete(input: { providerId: string }): Promise; diff --git a/packages/local-runtime-v2/src/application/session/process-local-application.ts b/packages/local-runtime-v2/src/application/session/process-local-application.ts index 3006d8823..1a2f877df 100644 --- a/packages/local-runtime-v2/src/application/session/process-local-application.ts +++ b/packages/local-runtime-v2/src/application/session/process-local-application.ts @@ -10,7 +10,10 @@ import type { } from '../../service/model-system/index.js'; import { watchGlobalEvents, watchProcessEvents } from '../events.js'; import type { ModelProviderApplication } from './model-provider-application.js'; -import type { LocalRuntimeApplication } from './process-local-application-contract.js'; +import type { + LocalRuntimeApplication, + ProcessLocalModelInput, +} from './process-local-application-contract.js'; import type { SessionReportCapability } from '../../service/session-system/index.js'; export interface ProcessLocalApplicationOptions { @@ -201,18 +204,7 @@ function toProviderRecord(provider: ModelProviderView): Record } function toUserModelInputs( - models: readonly { - modelId: string; - displayName?: string; - configurationSource?: string; - enabled?: boolean; - attachment?: boolean; - reasoning?: boolean; - toolCall?: boolean; - temperature?: boolean; - modalities?: { input?: readonly string[]; output?: readonly string[] }; - limit?: { context?: number; output?: number }; - }[], + models: readonly ProcessLocalModelInput[], ): UserModelInputView[] { return models.map(({ modalities, ...model }) => ({ ...model, diff --git a/packages/local-runtime-v2/src/service/model-system/contracts.ts b/packages/local-runtime-v2/src/service/model-system/contracts.ts index b9a56084b..30d3f3c54 100644 --- a/packages/local-runtime-v2/src/service/model-system/contracts.ts +++ b/packages/local-runtime-v2/src/service/model-system/contracts.ts @@ -414,6 +414,7 @@ export interface UserModelInputView { reasoning?: boolean; toolCall?: boolean; temperature?: boolean; + capabilities?: LocalModelConfig['capabilities']; modalities?: { input?: string[]; output?: string[] }; limit?: { context?: number; output?: number }; thinkingConfig?: { mode?: string; defaultValue?: string }; diff --git a/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts b/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts index 8fa267ef9..b1c6ed9b7 100644 --- a/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts +++ b/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts @@ -13,6 +13,31 @@ import { } from './service-input.js'; describe('model provider input normalization', () => { + it('preserves declared capabilities on create and merges updates without losing limits', () => { + const created = modelsFromInputs([ + { + modelId: 'vision-model', + capabilities: { support_image: true, max_image_bytes_inline: 1024 }, + }, + ]); + expect(created['vision-model']?.capabilities).toEqual({ + support_image: true, + max_image_bytes_inline: 1024, + }); + const patch = [{ modelId: 'vision-model', capabilities: { support_image: false } }]; + for (const updated of [modelsFromInputs(patch, created), mergeModelsFromInputs(created, patch)]) { + expect(updated['vision-model']?.capabilities).toEqual({ + support_image: false, + max_image_bytes_inline: 1024, + }); + const renamed = mergeModelsFromInputs(updated, [ + { modelId: 'vision-model', displayName: 'Renamed' }, + ]); + expect(renamed['vision-model']?.capabilities).toEqual(updated['vision-model']?.capabilities); + } + expect(created['vision-model']?.capabilities?.support_image).toBe(true); + }); + it('normalizes API keys and formats', () => { expect(() => assertValidRawApiKey(undefined as unknown as string)).toThrow( LocalModelProviderError, diff --git a/packages/local-runtime-v2/src/service/model-system/management/service-input.ts b/packages/local-runtime-v2/src/service/model-system/management/service-input.ts index 5f7baf346..e987cc40c 100644 --- a/packages/local-runtime-v2/src/service/model-system/management/service-input.ts +++ b/packages/local-runtime-v2/src/service/model-system/management/service-input.ts @@ -120,6 +120,9 @@ function modelFromInput( const nextModel: LocalModelConfig = { ...existing, ...modelMetadataFields(model), + ...(model.capabilities + ? { capabilities: { ...existing?.capabilities, ...model.capabilities } } + : {}), ...modelReasoningFields(model, effortOptions, defaultThinkingConfig), ...modelThinkingConfigFields(model, defaultThinkingConfig), ...modelEffortFields(effortOptions), diff --git a/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts b/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts index 6138755a0..3f5981a8c 100644 --- a/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts +++ b/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts @@ -626,6 +626,19 @@ describe('resolveThinkingLevel', () => { }); describe('model capability helpers', () => { + it('requires explicit image input metadata instead of generic attachment support', () => { + expect(capabilitiesFromModelConfig({ attachment: true }).support_image).toBe(false); + expect( + capabilitiesFromModelConfig({ modalities: { input: ['text'], output: ['image'] } }).support_image, + ).toBe(false); + expect( + capabilitiesFromModelConfig({ modalities: { input: ['text', 'image'] } }).support_image, + ).toBe(true); + expect( + capabilitiesFromModelConfig({ capabilities: { support_image: false } }).support_image, + ).toBe(false); + }); + it('uses explicit capability flags and forced thinking modes', () => { expect( capabilitiesFromModelConfig({ diff --git a/packages/tui/src/cli/program.ts b/packages/tui/src/cli/program.ts index f2e33a6cb..e3f831f68 100644 --- a/packages/tui/src/cli/program.ts +++ b/packages/tui/src/cli/program.ts @@ -194,6 +194,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { .option('--api-key-env ', 'environment variable containing the API key') .option('--context-limit ', 'context limit for every listed model', parsePositiveSafeInteger) .option('--output-limit ', 'output limit for every listed model', parsePositiveSafeInteger) + .option('--support-image', 'declare image input support for every listed model') .option('--use', 'test the first model, then save and select it as the default') .action( (commandOptions: { @@ -203,6 +204,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { model: string[]; contextLimit?: number; outputLimit?: number; + supportImage?: boolean; apiKeyEnv?: string; use?: boolean; }) => { @@ -217,6 +219,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { models: commandOptions.model, contextLimit: commandOptions.contextLimit, outputLimit: commandOptions.outputLimit, + supportImage: commandOptions.supportImage, apiKeyEnv: commandOptions.apiKeyEnv, saveAndUse: commandOptions.use, }); diff --git a/packages/tui/src/cli/provider-command.ts b/packages/tui/src/cli/provider-command.ts index a3ce4623f..01798da7b 100644 --- a/packages/tui/src/cli/provider-command.ts +++ b/packages/tui/src/cli/provider-command.ts @@ -14,6 +14,7 @@ export type McodeProviderCliRequest = readonly models: readonly string[]; readonly contextLimit?: number; readonly outputLimit?: number; + readonly supportImage?: boolean; readonly apiKeyEnv?: string; readonly saveAndUse?: boolean; } @@ -67,6 +68,7 @@ export async function runMcodeProviderCommand( apiFormat: request.apiFormat, models: request.models.map((modelId) => ({ modelId, + ...(request.supportImage ? { capabilities: { support_image: true } } : {}), ...(request.contextLimit !== undefined || request.outputLimit !== undefined ? { limit: { diff --git a/packages/tui/src/provider/contract.ts b/packages/tui/src/provider/contract.ts index df204e3af..cdaeff22b 100644 --- a/packages/tui/src/provider/contract.ts +++ b/packages/tui/src/provider/contract.ts @@ -1,3 +1,5 @@ +import type { ModelConfig } from '@mavis/config'; + export const MCODE_PROVIDER_API_FORMATS = [ 'anthropic-messages', 'openai-completions', @@ -74,6 +76,7 @@ export interface McodeProviderModelInput { readonly reasoning?: boolean; readonly toolCall?: boolean; readonly temperature?: boolean; + readonly capabilities?: Readonly>; readonly modalities?: { readonly input?: readonly string[]; readonly output?: readonly string[] }; readonly limit?: { readonly context?: number; readonly output?: number }; } diff --git a/test/byok.test.mjs b/test/byok.test.mjs index 78facd3b4..64dba7eed 100644 --- a/test/byok.test.mjs +++ b/test/byok.test.mjs @@ -298,6 +298,7 @@ test( "provider", "add", "--name", "Limited", "--base-url", baseUrl, "--api-format", "openai-completions", "--model", "fixture-model", "--model", "second-model", "--context-limit", "32768", "--output-limit", "4096", "--use", + "--support-image", ]; const beforeFailure = readFileSync(configPath, "utf8"); rejectConnection = true; @@ -312,6 +313,7 @@ test( assert.equal(config.defaultModel, "custom_provider:limited/fixture-model"); for (const model of ["fixture-model", "second-model"]) { assert.deepEqual(config.custom_provider.limited.models[model].limit, { context: 32768, output: 4096 }); + assert.deepEqual(config.custom_provider.limited.models[model].capabilities, { support_image: true }); } const configured = JSON.parse(await run(["provider", "list", "--json"])).providers.find( (provider) => provider.name === "Limited", @@ -330,8 +332,10 @@ test( await run([ "provider", "add", "--name", name, "--base-url", baseUrl, "--api-format", "openai-completions", "--model", "fixture-model", flag, "32768", + "--support-image", ]); assert.deepEqual(savedConfig().custom_provider[name].models["fixture-model"].limit, limit); + assert.deepEqual(savedConfig().custom_provider[name].models["fixture-model"].capabilities, { support_image: true }); assert.equal(savedConfig().defaultModel, config.defaultModel); } assert.equal(requests.length, beforeSaveOnly, "Limits alone must not test or select a model"); @@ -342,7 +346,48 @@ test( reasoning: true, thinking: { effortOptions: ["low", "high", "max"] }, }; + effortConfig.custom_provider.fixture.models["vision-modalities"] = { + modalities: { input: ["text", "image"], output: ["text"] }, + }; + effortConfig.custom_provider.fixture.models["attachment-only"] = { attachment: true }; writeFileSync(configPath, stringifyYaml(effortConfig)); + // The packaged command must carry the declaration through saving, a fresh + // process/config reload, attachment preparation and SDK serialization. + const imageBase64 = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR4nGP4n8LwnxLMMGrAqAGjBgwXAwCOqWIfmQV0zAAAAABJRU5ErkJggg=="; + const imagePath = path.join(workspaceDir, "fixture.png"); + writeFileSync(imagePath, Buffer.from(imageBase64, "base64")); + for (const [model, supportsImage] of [ + ["custom_provider:limited/fixture-model", true], + ["custom_provider:context-only/fixture-model", true], + ["custom_provider:fixture/vision-modalities", true], + ["custom_provider:fixture/fixture-model", false], + ["custom_provider:fixture/attachment-only", false], + ]) { + const beforeImage = requests.length; + await run(["exec", "IMAGE_INPUT_TEST", "--model", model, "--file", imagePath, + "--timeout", "20s", "--max-steps", "1"]); + const turns = requests.slice(beforeImage).filter((r) => r.body.stream === true); + assert.ok(turns.length > 0, `Expected a runtime request for ${model}`); + for (const { body } of turns) { + const images = body.messages.flatMap((message) => + Array.isArray(message.content) + ? message.content.filter((part) => part.type === "image_url") + : []); + if (supportsImage) { + assert.equal(images.length, 1, `Expected an inline image for ${model}`); + // PNG inputs are transcoded to JPEG by the real attachment pipeline. + const url = images[0].image_url.url; + assert.match(url, /^data:image\/jpeg;base64,/); + const bytes = Buffer.from(url.split(",")[1], "base64"); + assert.deepEqual([...bytes.subarray(0, 3)], [0xff, 0xd8, 0xff]); + assert.deepEqual([...bytes.subarray(-2)], [0xff, 0xd9]); + assert.doesNotMatch(JSON.stringify(body.messages), /current model cannot read this image inline/); + } else { + assert.deepEqual(images, [], `${model} must remain text-only`); + assert.match(JSON.stringify(body.messages), /current model cannot read this image inline/); + } + } + } const effortModel = `${selected.providerId}/kimi-k3`; for (const effort of ["low", "high", "max"]) { const beforeEffort = requests.length;