Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions agentic-flow/src/router/providers/onnx-local-optimized.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,18 @@
* - Better generation parameters for code tasks
* - System prompt caching
*
* Note: onnxruntime-node is optional - will error if not installed
* Note: onnxruntime-node is optional - will error if not installed.
*
* NOTE (ruvnet/ruflo#2048): the previous top-level `await import('onnxruntime-node')`
* fired the native-binding load (`onnxruntime_binding.node`) at module
* import time. On Windows this crashes with "OS cannot run %1" — and the
* crash propagated to any consumer that transitively imports this file
* (e.g. `agentic-flow/reasoningbank` via `core/distill -> router/router`).
* This file does not use `ort` directly — the base `ONNXLocalProvider`
* it extends does, and that file now lazy-loads ort on first session
* init. So we just drop the eager top-level load here.
*/

let ort: any = null;

// Dynamic import for optional onnxruntime-node
try {
ort = await import('onnxruntime-node');
} catch {
// Will be handled at runtime
}

import { get_encoding } from 'tiktoken';
import { ensurePhi4Model, ModelDownloader, PHI4_MODEL_PATH } from '../../utils/model-downloader.js';
import type {
Expand Down
28 changes: 21 additions & 7 deletions agentic-flow/src/router/providers/onnx-local.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,30 @@
*
* Uses onnxruntime-node for true local CPU/GPU inference
* Falls back gracefully when native module isn't available (Windows)
*
* NOTE (ruvnet/ruflo#2048): `onnxruntime-node` is loaded LAZILY on first
* `initializeSession()` call, not at module import. The previous top-level
* `await import('onnxruntime-node')` fired the native-binding load
* (`onnxruntime_binding.node`) at module load time, which crashed Windows
* environments where the NAPI binary cannot be loaded — even when the
* consumer (e.g. `agentic-flow/reasoningbank`) never actually invokes
* the router. Moving the import inside `loadOrt()` keeps importing
* `reasoningbank` side-effect-free with respect to native bindings.
*/

let ort: any = null;
let ortAvailable = false;

// Dynamic import for optional onnxruntime-node
try {
ort = await import('onnxruntime-node');
ortAvailable = true;
} catch {
console.warn('[ONNX] onnxruntime-node not available - local inference disabled');
let ortLoaded = false;

async function loadOrt(): Promise<void> {
if (ortLoaded) return;
ortLoaded = true;
try {
ort = await import('onnxruntime-node');
ortAvailable = true;
} catch {
console.warn('[ONNX] onnxruntime-node not available - local inference disabled');
}
}

import * as fs from 'fs';
Expand Down Expand Up @@ -107,6 +120,7 @@ export class ONNXLocalProvider implements LLMProvider {
private async initializeSession(): Promise<void> {
if (this.session) return;

await loadOrt();
if (!ortAvailable || !ort) {
throw new Error('onnxruntime-node not available - install with: npm install onnxruntime-node');
}
Expand Down
Loading