diff --git a/docs/examples.md b/docs/examples.md index 12a536569..748cad8a4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -70,6 +70,41 @@ pnpm mcode provider list --json ## 3. Search and image input +For a custom BYOK model, declare image input support explicitly when adding the +provider. Use this only if the selected provider endpoint and model accept images: + +```bash +pnpm mcode provider add --name my-vision-provider --base-url https://example.com/v1 \ + --api-format openai-completions --model my-vision-model \ + --api-key-env MCODE_PROVIDER_API_KEY --support-image --use +``` + +`--support-image` applies to every repeated `--model` and saves +`capabilities.support_image: true`. Without it, adding an unknown custom model +does not infer image support from its name. `--use` and `provider test` check +connectivity with a text request; they do not verify vision support. + +For an existing provider, close MCode and add the capability to the matching model +in the active profile's `config.yaml` (normally `~/.minimax/config.yaml`; see +[Accounts and data](installation.md#accounts-and-data) for profiles and overrides). +Merge this fragment into the existing provider and keep its other settings: + +```yaml +custom_provider: + my-vision-provider: + models: + my-vision-model: + capabilities: + support_image: true +``` + +The existing `modalities: { input: [text, image], output: [text] }` model setting +also enables image input. Either declaration is sufficient. To make a model +text-only again, remove `image` from `modalities.input` and remove or set +`capabilities.support_image` to `false`. Generic `attachment: true` alone does not +declare image input support. Restart MCode after editing configuration and select +the configured model with `/provider`, or use `exec --model` for a single run. + After signing in to MiniMax, try a task that explicitly requires search: > Use web_search to find the official Node.js test runner documentation. Summarize how to run tests and include the source URL. If the tool is unavailable, say so. diff --git a/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts b/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts index f000d26ff..ff803b1ae 100644 --- a/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts +++ b/packages/local-runtime-v2/src/application/session/process-local-application-contract.ts @@ -26,8 +26,13 @@ import type { CodexOAuthStartResult, CodexOAuthLoginOptions, CodexOAuthStatus, + UserModelInputView, } from "../../service/model-system/index.js"; +export type ProcessLocalModelInput = Omit & { + modalities?: { input?: readonly string[]; output?: readonly string[] }; +}; + export interface ProcessLocalMiniAppLaunch { readonly miniApp: MiniAppSurfaceSummary; /** Main-process-only launch coordinates. Never forward this object to a renderer. */ @@ -262,7 +267,7 @@ export interface LocalRuntimeApplication { baseUrl: string; apiKey: string; apiFormat?: string; - models?: readonly { modelId: string; displayName?: string }[]; + models?: readonly ProcessLocalModelInput[]; saveAndUse?: boolean; }): Promise; discoverCandidate(input: { @@ -278,21 +283,7 @@ export interface LocalRuntimeApplication { baseUrl: string; apiKey?: string; apiFormat?: string; - models?: readonly { - modelId: string; - displayName?: string; - configurationSource?: string; - enabled?: boolean; - attachment?: boolean; - reasoning?: boolean; - toolCall?: boolean; - temperature?: boolean; - modalities?: { - input?: readonly string[]; - output?: readonly string[]; - }; - limit?: { context?: number; output?: number }; - }[]; + models?: readonly ProcessLocalModelInput[]; }; modelId?: string; saveAndUse?: boolean; @@ -309,7 +300,7 @@ export interface LocalRuntimeApplication { apiKey?: string; apiFormat?: string; enabled?: boolean; - models?: readonly { modelId: string; displayName?: string }[]; + models?: readonly ProcessLocalModelInput[]; saveAndUse?: boolean; }): Promise; delete(input: { providerId: string }): Promise; diff --git a/packages/local-runtime-v2/src/application/session/process-local-application.ts b/packages/local-runtime-v2/src/application/session/process-local-application.ts index 3006d8823..1a2f877df 100644 --- a/packages/local-runtime-v2/src/application/session/process-local-application.ts +++ b/packages/local-runtime-v2/src/application/session/process-local-application.ts @@ -10,7 +10,10 @@ import type { } from '../../service/model-system/index.js'; import { watchGlobalEvents, watchProcessEvents } from '../events.js'; import type { ModelProviderApplication } from './model-provider-application.js'; -import type { LocalRuntimeApplication } from './process-local-application-contract.js'; +import type { + LocalRuntimeApplication, + ProcessLocalModelInput, +} from './process-local-application-contract.js'; import type { SessionReportCapability } from '../../service/session-system/index.js'; export interface ProcessLocalApplicationOptions { @@ -201,18 +204,7 @@ function toProviderRecord(provider: ModelProviderView): Record } function toUserModelInputs( - models: readonly { - modelId: string; - displayName?: string; - configurationSource?: string; - enabled?: boolean; - attachment?: boolean; - reasoning?: boolean; - toolCall?: boolean; - temperature?: boolean; - modalities?: { input?: readonly string[]; output?: readonly string[] }; - limit?: { context?: number; output?: number }; - }[], + models: readonly ProcessLocalModelInput[], ): UserModelInputView[] { return models.map(({ modalities, ...model }) => ({ ...model, diff --git a/packages/local-runtime-v2/src/service/model-system/contracts.ts b/packages/local-runtime-v2/src/service/model-system/contracts.ts index b9a56084b..30d3f3c54 100644 --- a/packages/local-runtime-v2/src/service/model-system/contracts.ts +++ b/packages/local-runtime-v2/src/service/model-system/contracts.ts @@ -414,6 +414,7 @@ export interface UserModelInputView { reasoning?: boolean; toolCall?: boolean; temperature?: boolean; + capabilities?: LocalModelConfig['capabilities']; modalities?: { input?: string[]; output?: string[] }; limit?: { context?: number; output?: number }; thinkingConfig?: { mode?: string; defaultValue?: string }; diff --git a/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts b/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts index 8fa267ef9..b1c6ed9b7 100644 --- a/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts +++ b/packages/local-runtime-v2/src/service/model-system/management/service-input.test.ts @@ -13,6 +13,31 @@ import { } from './service-input.js'; describe('model provider input normalization', () => { + it('preserves declared capabilities on create and merges updates without losing limits', () => { + const created = modelsFromInputs([ + { + modelId: 'vision-model', + capabilities: { support_image: true, max_image_bytes_inline: 1024 }, + }, + ]); + expect(created['vision-model']?.capabilities).toEqual({ + support_image: true, + max_image_bytes_inline: 1024, + }); + const patch = [{ modelId: 'vision-model', capabilities: { support_image: false } }]; + for (const updated of [modelsFromInputs(patch, created), mergeModelsFromInputs(created, patch)]) { + expect(updated['vision-model']?.capabilities).toEqual({ + support_image: false, + max_image_bytes_inline: 1024, + }); + const renamed = mergeModelsFromInputs(updated, [ + { modelId: 'vision-model', displayName: 'Renamed' }, + ]); + expect(renamed['vision-model']?.capabilities).toEqual(updated['vision-model']?.capabilities); + } + expect(created['vision-model']?.capabilities?.support_image).toBe(true); + }); + it('normalizes API keys and formats', () => { expect(() => assertValidRawApiKey(undefined as unknown as string)).toThrow( LocalModelProviderError, diff --git a/packages/local-runtime-v2/src/service/model-system/management/service-input.ts b/packages/local-runtime-v2/src/service/model-system/management/service-input.ts index 5f7baf346..e987cc40c 100644 --- a/packages/local-runtime-v2/src/service/model-system/management/service-input.ts +++ b/packages/local-runtime-v2/src/service/model-system/management/service-input.ts @@ -120,6 +120,9 @@ function modelFromInput( const nextModel: LocalModelConfig = { ...existing, ...modelMetadataFields(model), + ...(model.capabilities + ? { capabilities: { ...existing?.capabilities, ...model.capabilities } } + : {}), ...modelReasoningFields(model, effortOptions, defaultThinkingConfig), ...modelThinkingConfigFields(model, defaultThinkingConfig), ...modelEffortFields(effortOptions), diff --git a/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts b/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts index 6138755a0..3f5981a8c 100644 --- a/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts +++ b/packages/local-runtime-v2/src/service/model-system/resolution/model-ref.test.ts @@ -626,6 +626,19 @@ describe('resolveThinkingLevel', () => { }); describe('model capability helpers', () => { + it('requires explicit image input metadata instead of generic attachment support', () => { + expect(capabilitiesFromModelConfig({ attachment: true }).support_image).toBe(false); + expect( + capabilitiesFromModelConfig({ modalities: { input: ['text'], output: ['image'] } }).support_image, + ).toBe(false); + expect( + capabilitiesFromModelConfig({ modalities: { input: ['text', 'image'] } }).support_image, + ).toBe(true); + expect( + capabilitiesFromModelConfig({ capabilities: { support_image: false } }).support_image, + ).toBe(false); + }); + it('uses explicit capability flags and forced thinking modes', () => { expect( capabilitiesFromModelConfig({ diff --git a/packages/tui/src/cli/program.ts b/packages/tui/src/cli/program.ts index f2e33a6cb..e3f831f68 100644 --- a/packages/tui/src/cli/program.ts +++ b/packages/tui/src/cli/program.ts @@ -194,6 +194,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { .option('--api-key-env ', 'environment variable containing the API key') .option('--context-limit ', 'context limit for every listed model', parsePositiveSafeInteger) .option('--output-limit ', 'output limit for every listed model', parsePositiveSafeInteger) + .option('--support-image', 'declare image input support for every listed model') .option('--use', 'test the first model, then save and select it as the default') .action( (commandOptions: { @@ -203,6 +204,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { model: string[]; contextLimit?: number; outputLimit?: number; + supportImage?: boolean; apiKeyEnv?: string; use?: boolean; }) => { @@ -217,6 +219,7 @@ export function createTuiProgram(options: CreateTuiProgramOptions): Command { models: commandOptions.model, contextLimit: commandOptions.contextLimit, outputLimit: commandOptions.outputLimit, + supportImage: commandOptions.supportImage, apiKeyEnv: commandOptions.apiKeyEnv, saveAndUse: commandOptions.use, }); diff --git a/packages/tui/src/cli/provider-command.ts b/packages/tui/src/cli/provider-command.ts index a3ce4623f..01798da7b 100644 --- a/packages/tui/src/cli/provider-command.ts +++ b/packages/tui/src/cli/provider-command.ts @@ -14,6 +14,7 @@ export type McodeProviderCliRequest = readonly models: readonly string[]; readonly contextLimit?: number; readonly outputLimit?: number; + readonly supportImage?: boolean; readonly apiKeyEnv?: string; readonly saveAndUse?: boolean; } @@ -67,6 +68,7 @@ export async function runMcodeProviderCommand( apiFormat: request.apiFormat, models: request.models.map((modelId) => ({ modelId, + ...(request.supportImage ? { capabilities: { support_image: true } } : {}), ...(request.contextLimit !== undefined || request.outputLimit !== undefined ? { limit: { diff --git a/packages/tui/src/provider/contract.ts b/packages/tui/src/provider/contract.ts index df204e3af..cdaeff22b 100644 --- a/packages/tui/src/provider/contract.ts +++ b/packages/tui/src/provider/contract.ts @@ -1,3 +1,5 @@ +import type { ModelConfig } from '@mavis/config'; + export const MCODE_PROVIDER_API_FORMATS = [ 'anthropic-messages', 'openai-completions', @@ -74,6 +76,7 @@ export interface McodeProviderModelInput { readonly reasoning?: boolean; readonly toolCall?: boolean; readonly temperature?: boolean; + readonly capabilities?: Readonly>; readonly modalities?: { readonly input?: readonly string[]; readonly output?: readonly string[] }; readonly limit?: { readonly context?: number; readonly output?: number }; } diff --git a/test/byok.test.mjs b/test/byok.test.mjs index 78facd3b4..64dba7eed 100644 --- a/test/byok.test.mjs +++ b/test/byok.test.mjs @@ -298,6 +298,7 @@ test( "provider", "add", "--name", "Limited", "--base-url", baseUrl, "--api-format", "openai-completions", "--model", "fixture-model", "--model", "second-model", "--context-limit", "32768", "--output-limit", "4096", "--use", + "--support-image", ]; const beforeFailure = readFileSync(configPath, "utf8"); rejectConnection = true; @@ -312,6 +313,7 @@ test( assert.equal(config.defaultModel, "custom_provider:limited/fixture-model"); for (const model of ["fixture-model", "second-model"]) { assert.deepEqual(config.custom_provider.limited.models[model].limit, { context: 32768, output: 4096 }); + assert.deepEqual(config.custom_provider.limited.models[model].capabilities, { support_image: true }); } const configured = JSON.parse(await run(["provider", "list", "--json"])).providers.find( (provider) => provider.name === "Limited", @@ -330,8 +332,10 @@ test( await run([ "provider", "add", "--name", name, "--base-url", baseUrl, "--api-format", "openai-completions", "--model", "fixture-model", flag, "32768", + "--support-image", ]); assert.deepEqual(savedConfig().custom_provider[name].models["fixture-model"].limit, limit); + assert.deepEqual(savedConfig().custom_provider[name].models["fixture-model"].capabilities, { support_image: true }); assert.equal(savedConfig().defaultModel, config.defaultModel); } assert.equal(requests.length, beforeSaveOnly, "Limits alone must not test or select a model"); @@ -342,7 +346,48 @@ test( reasoning: true, thinking: { effortOptions: ["low", "high", "max"] }, }; + effortConfig.custom_provider.fixture.models["vision-modalities"] = { + modalities: { input: ["text", "image"], output: ["text"] }, + }; + effortConfig.custom_provider.fixture.models["attachment-only"] = { attachment: true }; writeFileSync(configPath, stringifyYaml(effortConfig)); + // The packaged command must carry the declaration through saving, a fresh + // process/config reload, attachment preparation and SDK serialization. + const imageBase64 = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGUlEQVR4nGP4n8LwnxLMMGrAqAGjBgwXAwCOqWIfmQV0zAAAAABJRU5ErkJggg=="; + const imagePath = path.join(workspaceDir, "fixture.png"); + writeFileSync(imagePath, Buffer.from(imageBase64, "base64")); + for (const [model, supportsImage] of [ + ["custom_provider:limited/fixture-model", true], + ["custom_provider:context-only/fixture-model", true], + ["custom_provider:fixture/vision-modalities", true], + ["custom_provider:fixture/fixture-model", false], + ["custom_provider:fixture/attachment-only", false], + ]) { + const beforeImage = requests.length; + await run(["exec", "IMAGE_INPUT_TEST", "--model", model, "--file", imagePath, + "--timeout", "20s", "--max-steps", "1"]); + const turns = requests.slice(beforeImage).filter((r) => r.body.stream === true); + assert.ok(turns.length > 0, `Expected a runtime request for ${model}`); + for (const { body } of turns) { + const images = body.messages.flatMap((message) => + Array.isArray(message.content) + ? message.content.filter((part) => part.type === "image_url") + : []); + if (supportsImage) { + assert.equal(images.length, 1, `Expected an inline image for ${model}`); + // PNG inputs are transcoded to JPEG by the real attachment pipeline. + const url = images[0].image_url.url; + assert.match(url, /^data:image\/jpeg;base64,/); + const bytes = Buffer.from(url.split(",")[1], "base64"); + assert.deepEqual([...bytes.subarray(0, 3)], [0xff, 0xd8, 0xff]); + assert.deepEqual([...bytes.subarray(-2)], [0xff, 0xd9]); + assert.doesNotMatch(JSON.stringify(body.messages), /current model cannot read this image inline/); + } else { + assert.deepEqual(images, [], `${model} must remain text-only`); + assert.match(JSON.stringify(body.messages), /current model cannot read this image inline/); + } + } + } const effortModel = `${selected.providerId}/kimi-k3`; for (const effort of ["low", "high", "max"]) { const beforeEffort = requests.length;