diff --git a/config.template.jsonc b/config.template.jsonc index f64215def8..145a343dd3 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -341,6 +341,10 @@ "apiKey": "", "apiBaseUrl": "https://llm.onerouter.pro/v1" }, + "meta": { + "apiKey": "", + "apiBaseUrl": "https://api.meta.ai/v1" + }, "together-ai": { "apiKey": "" }, // Local Ollama. `enabled: false` skips the auto-probe at startup // (otherwise Puter logs ECONNREFUSED on every boot when no Ollama diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index 2336365236..1b18e7815f 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -38,6 +38,7 @@ import { FakeChatProvider } from './providers/FakeChatProvider.js'; import { GeminiChatProvider } from './providers/gemini/GeminiChatProvider.js'; import { GroqAIProvider } from './providers/groq/GroqAIProvider.js'; import { InfronProvider } from './providers/infron/InfronProvider.js'; +import { MetaProvider } from './providers/meta/MetaProvider.js'; import { MiniMaxProvider } from './providers/minimax/MiniMaxProvider.js'; import { MistralAIProvider } from './providers/mistral/MistralAiProvider.js'; import { MoonshotProvider } from './providers/moonshot/MoonshotProvider.js'; @@ -1045,6 +1046,18 @@ export class ChatCompletionDriver extends PuterDriver { ); } + const meta = providers['meta']; + const metaKey = readKey(meta); + if (metaKey) { + this.#providers['meta'] = new MetaProvider( + { + apiKey: metaKey, + apiBaseUrl: meta?.apiBaseUrl as string | undefined, + }, + metering, + ); + } + const neuralwatt = providers['neuralwatt']; const neuralwattKey = readKey(neuralwatt); if (neuralwattKey) { diff --git a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.test.ts b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.test.ts new file mode 100644 index 0000000000..f847dde784 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import { SYSTEM_ACTOR } from '../../../../core/actor.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { META_MODELS } from './models.js'; +import { MetaProvider } from './MetaProvider.js'; + +const { createMock, openAICtor } = vi.hoisted(() => { + const createMock = vi.fn(); + const openAICtor = vi.fn(); + return { createMock, openAICtor }; +}); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.chat = { completions: { create: createMock } }; + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +let server: PuterServer; +let recordSpy: MockInstance; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = () => { + const provider = new MetaProvider( + { apiKey: 'test-meta-key' }, + server.services.metering, + ); + return { provider }; +}; + +beforeEach(() => { + createMock.mockReset(); + openAICtor.mockReset(); + recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('MetaProvider', () => { + it('points the OpenAI SDK at the Meta base URL with the configured key', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledTimes(1); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-meta-key', + baseURL: 'https://api.meta.ai/v1', + }); + }); + + it('returns available models', () => { + const { provider } = makeProvider(); + const models = provider.models(); + expect(models.length).toBeGreaterThan(0); + expect(models.some((m) => m.id === 'meta:muse-spark-1.2')).toBe(true); + }); + + it('returns default model as meta:muse-spark-1.2', () => { + const { provider } = makeProvider(); + expect(provider.getDefaultModel()).toBe('meta:muse-spark-1.2'); + }); + + it('returns model list and aliases', async () => { + const { provider } = makeProvider(); + const list = await provider.list(); + expect(list).toContain('meta:muse-spark-1.2'); + expect(list).toContain('muse-spark-1.2'); + }); + + it('completes chat request and calculates usage costs correctly using cost keys', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'hello from muse', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'muse-spark-1.2', + messages: [{ role: 'user', content: 'hello' }], + }), + ); + + expect(result).toMatchObject({ + message: { content: 'hello from muse', role: 'assistant' }, + finish_reason: 'stop', + }); + + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, actor, prefix, costsOverride] = recordSpy.mock.calls[0]!; + expect(usage).toEqual({ + prompt_tokens: 10, + completion_tokens: 5, + cached_tokens: 0, + }); + expect(actor).toBe(SYSTEM_ACTOR); + expect(prefix).toBe('meta:muse-spark-1.2'); + // prompt_tokens: 10 * 125 = 1250, completion_tokens: 5 * 425 = 2125 + expect(costsOverride).toEqual({ + prompt_tokens: 1250, + completion_tokens: 2125, + cached_tokens: 0, + }); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts new file mode 100644 index 0000000000..b71e5c00ed --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources/index.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatProvider, + ICompleteArguments, + IChatCompleteResult, +} from '../../types.js'; +import { META_MODELS } from './models.js'; + +export class MetaProvider implements IChatProvider { + #openai: OpenAI; + + #meteringService: MeteringService; + + constructor( + config: { apiKey: string; apiBaseUrl?: string }, + meteringService: MeteringService, + ) { + this.#openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: config.apiBaseUrl || 'https://api.meta.ai/v1', + }); + this.#meteringService = meteringService; + } + + getDefaultModel() { + return 'meta:muse-spark-1.2'; + } + + models() { + return META_MODELS; + } + + async list() { + const models = this.models(); + const modelNames: string[] = []; + for (const model of models) { + modelNames.push(model.id); + if (model.aliases) { + modelNames.push(...model.aliases); + } + } + return modelNames; + } + + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + }: ICompleteArguments): Promise { + const actor = Context.get('actor'); + const availableModels = this.models(); + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; + + const modelIdForParams = modelUsed.id.startsWith('meta:') + ? modelUsed.id.slice('meta:'.length) + : modelUsed.id; + + messages = await OpenAIUtil.process_input_messages(messages); + let completion; + try { + completion = await this.#openai.chat.completions.create({ + messages, + model: modelIdForParams, + ...(tools ? { tools } : {}), + max_tokens, + temperature, + stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + } as ChatCompletionCreateParams); + } catch (e) { + console.log('Meta API completion error: ', e); + throw e; + } + + return OpenAIUtil.handle_completion_output({ + usage_calculator: ({ usage }) => { + const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); + const inputKey = + (modelUsed.input_cost_key as string) || 'prompt_tokens'; + const outputKey = + (modelUsed.output_cost_key as string) || + 'completion_tokens'; + + const costsOverride = { + prompt_tokens: + (trackedUsage.prompt_tokens ?? 0) * + Number(modelUsed.costs[inputKey] ?? 0), + completion_tokens: + (trackedUsage.completion_tokens ?? 0) * + Number(modelUsed.costs[outputKey] ?? 0), + cached_tokens: + (trackedUsage.cached_tokens ?? 0) * + Number(modelUsed.costs.cached_tokens ?? 0), + }; + + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor, + modelUsed.id, + costsOverride, + ); + return trackedUsage; + }, + stream, + completion, + }); + } + + checkModeration( + _text: string, + ): ReturnType { + throw new Error('Method not implemented.'); + } +} diff --git a/src/backend/drivers/ai-chat/providers/meta/models.ts b/src/backend/drivers/ai-chat/providers/meta/models.ts new file mode 100644 index 0000000000..ba3cf79667 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/meta/models.ts @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IChatModel } from '../../types.js'; + +export const META_MODELS: IChatModel[] = [ + { + puterId: 'meta:meta/muse-spark-1.2', + id: 'meta:muse-spark-1.2', + aliases: ['meta/muse-spark-1.2', 'muse-spark-1.2'], + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + context: 1048576, + max_tokens: 8192, + costs: { + tokens: 1_000_000, + prompt_tokens: 125, // micro-cents per token + completion_tokens: 425, // micro-cents per token + cached_tokens: 0, + }, + }, + { + puterId: 'meta:meta/muse-spark-1.1', + id: 'meta:muse-spark-1.1', + aliases: ['meta/muse-spark-1.1', 'muse-spark-1.1'], + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + context: 1048576, + max_tokens: 8192, + costs: { + tokens: 1_000_000, + prompt_tokens: 125, + completion_tokens: 425, + cached_tokens: 0, + }, + }, + { + puterId: 'meta:meta/muse-code-1.0', + id: 'meta:muse-code-1.0', + aliases: ['meta/muse-code-1.0', 'muse-code-1.0'], + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + context: 1048576, + max_tokens: 8192, + costs: { + tokens: 1_000_000, + prompt_tokens: 150, + completion_tokens: 500, + cached_tokens: 0, + }, + }, +];