From b897b44b6508ad0da5beb0dd3bfc7a682623a61d Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 6 May 2026 17:57:56 +0800 Subject: [PATCH 1/5] native dependency --- install.sh | 18 +++++- package.json | 6 +- scripts/ensure-native-deps.cjs | 100 +++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 scripts/ensure-native-deps.cjs diff --git a/install.sh b/install.sh index 59a3c5b..0bccf0a 100644 --- a/install.sh +++ b/install.sh @@ -222,11 +222,19 @@ deploy_from_repo() { if [[ -f "README.md" ]]; then cp README.md "${PLUGIN_DEST}/" || true fi + if [[ -d "scripts" ]]; then + rm -rf "${PLUGIN_DEST}/scripts" + cp -R scripts "${PLUGIN_DEST}/" + fi info "Installing plugin dependencies..." - (cd "${PLUGIN_DEST}" && npm install --no-audit --no-fund) || { + (cd "${PLUGIN_DEST}" && npm install --no-audit --no-fund --ignore-scripts=false) || { err "npm install failed in ${PLUGIN_DEST}" exit 1 } + (cd "${PLUGIN_DEST}" && npm run native:verify) || { + err "native dependency verification failed in ${PLUGIN_DEST}" + exit 1 + } info "Plugin deployed: ${PLUGIN_DEST}" } @@ -243,6 +251,7 @@ deploy_from_github() { "openclaw-powermem-env.ts" "openclaw.plugin.json" "package.json" + "scripts/ensure-native-deps.cjs" "tsconfig.json" ".gitignore" ) @@ -250,6 +259,7 @@ deploy_from_github() { info "Downloading plugin from ${REPO}@${BRANCH}..." for f in "${files[@]}"; do local url="${gh_raw}/${f}" + mkdir -p "$(dirname "${PLUGIN_DEST}/${f}")" if curl -fsSL --connect-timeout 15 --max-time 60 -o "${PLUGIN_DEST}/${f}" "${url}" 2>/dev/null; then echo " ${f} ✓" else @@ -260,10 +270,14 @@ deploy_from_github() { fi done info "Installing plugin dependencies..." - (cd "${PLUGIN_DEST}" && npm install --no-audit --no-fund) || { + (cd "${PLUGIN_DEST}" && npm install --no-audit --no-fund --ignore-scripts=false) || { err "npm install failed in ${PLUGIN_DEST}" exit 1 } + (cd "${PLUGIN_DEST}" && npm run native:verify) || { + err "native dependency verification failed in ${PLUGIN_DEST}" + exit 1 + } info "Plugin deployed: ${PLUGIN_DEST}" } diff --git a/package.json b/package.json index 5e96318..8e5cc04 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,12 @@ "files": [ "src", "lib", + "scripts", "openclaw.plugin.json" ], "scripts": { + "postinstall": "node scripts/ensure-native-deps.cjs", + "native:verify": "node scripts/ensure-native-deps.cjs", "test": "vitest run", "lint": "tsc --noEmit" }, @@ -46,7 +49,8 @@ "license": "Apache-2.0", "pnpm": { "onlyBuiltDependencies": [ - "better-sqlite3" + "better-sqlite3", + "sqlite-vec" ] } } diff --git a/scripts/ensure-native-deps.cjs b/scripts/ensure-native-deps.cjs new file mode 100644 index 0000000..325d6b0 --- /dev/null +++ b/scripts/ensure-native-deps.cjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +"use strict"; + +const { spawnSync } = require("node:child_process"); +const path = require("node:path"); + +const rootDir = path.resolve(__dirname, ".."); +const nativePackages = ["better-sqlite3", "sqlite-vec"]; + +function log(message) { + console.log(`[memory-powermem] ${message}`); +} + +function warn(message) { + console.warn(`[memory-powermem] ${message}`); +} + +function npmCommand() { + return process.platform === "win32" ? "npm.cmd" : "npm"; +} + +function tryRequire(packageName) { + try { + const resolved = require.resolve(packageName, { paths: [rootDir] }); + require(resolved); + return null; + } catch (err) { + return err; + } +} + +function verifyNativePackages() { + const failures = []; + for (const packageName of nativePackages) { + const err = tryRequire(packageName); + if (err) { + failures.push({ packageName, err }); + } + } + return failures; +} + +function printFailures(failures) { + for (const { packageName, err } of failures) { + warn(`${packageName} failed to load: ${err && err.message ? err.message : String(err)}`); + } +} + +function rebuildNativePackages() { + log(`rebuilding native dependencies: ${nativePackages.join(", ")}`); + return spawnSync( + npmCommand(), + ["rebuild", ...nativePackages, "--build-from-source"], + { + cwd: rootDir, + env: { + ...process.env, + npm_config_ignore_scripts: "false", + }, + stdio: "inherit", + }, + ); +} + +function main() { + if (process.env.MEMORY_POWERMEM_SKIP_NATIVE_REBUILD === "1") { + warn("skipping native dependency verification because MEMORY_POWERMEM_SKIP_NATIVE_REBUILD=1"); + return; + } + + const initialFailures = verifyNativePackages(); + if (initialFailures.length === 0) { + log("native dependencies verified"); + return; + } + + printFailures(initialFailures); + const result = rebuildNativePackages(); + if (result.error) { + warn(`failed to run npm rebuild: ${result.error.message}`); + process.exit(1); + } + if (result.status !== 0) { + warn("native dependency rebuild failed"); + warn("install build tools first, then reinstall or run: npm rebuild better-sqlite3 sqlite-vec --build-from-source"); + warn("Debian/Ubuntu example: apt-get update && apt-get install -y python3 make gcc g++"); + process.exit(result.status ?? 1); + } + + const finalFailures = verifyNativePackages(); + if (finalFailures.length > 0) { + printFailures(finalFailures); + warn("native dependencies still failed after rebuild"); + process.exit(1); + } + + log("native dependencies rebuilt and verified"); +} + +main(); From 06cb2d64da205d49c8bc9fc99a7bce356a1c821c Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 6 May 2026 21:24:18 +0800 Subject: [PATCH 2/5] Import existing OpenClaw markdown memories into PowerMem with startup and CLI controls, per-file status tracking, and throttling safeguards. --- README.md | 8 + README_CN.md | 8 + openclaw.plugin.json | 44 ++ .../config-reference.md | 8 + src/config.ts | 71 +++ src/index.ts | 189 ++++++- src/markdown-import.ts | 501 ++++++++++++++++++ test/config.test.ts | 23 + test/markdown-import.test.ts | 149 ++++++ 9 files changed, 1000 insertions(+), 1 deletion(-) create mode 100644 src/markdown-import.ts create mode 100644 test/markdown-import.test.ts diff --git a/README.md b/README.md index ab01e4c..cf77436 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,12 @@ After installing, uninstalling, or changing config, restart the OpenClaw gateway | `autoExperience` | No | Auto-extract procedural experiences via LLM; default `true`. | | `experienceRecall` | No | Include experiences in recall results; default `true`. | | `inferOnAdd` | No | Use PowerMem intelligent extraction when adding; default `true`. | +| `importMarkdownOnStart` | No | One-time import of existing OpenClaw markdown memories on startup; default `false`. | +| `importMarkdownPaths` | No | Markdown files/directories to import. Defaults to `memory/`, `MEMORY.md`, and `USER.md`; relative paths resolve from the OpenClaw workspace. | +| `importMarkdownMaxFileBytes` | No | Max size for a single markdown file; default `10485760` (10 MiB). Larger files are marked `skipped_too_large`. | +| `importMarkdownBatchDelayMs` | No | Delay between imported chunks to avoid write bursts; default `300`. | +| `importMarkdownMaxFiles` | No | Optional hard cap on markdown files imported in one run. Empty means no cap. | +| `importMarkdownMaxChunks` | No | Optional hard cap on markdown chunks imported in one run. Empty means no cap. | | `dualWrite` | No | HTTP only: write to remote + local SQLite and queue failed writes. | | `dualWritePriority` | No | Dual-write priority: `"remote"` (default) tries PowerMem first and falls back to local SQLite; `"local"` writes/searches SQLite first and syncs to remote. | | `localDbPath` | No | Local SQLite path for `dualWrite`. | @@ -341,6 +347,8 @@ Exposed to OpenClaw agents: - `openclaw ltm search [--limit n]` — Search memories. - `openclaw ltm health` — Check PowerMem service health. - `openclaw ltm add ""` — Manually store one memory. +- `openclaw ltm import-md [paths...] [--force] [--dry-run] [--delay-ms n] [--max-file-bytes n] [--max-files n] [--max-chunks n]` — Import existing markdown memories. With no paths, scans `memory/`, `MEMORY.md`, and `USER.md`. +- `openclaw ltm import-md-status [paths...] [--json]` — Show per-file markdown import status: imported, changed, skipped, failed, or not imported. --- diff --git a/README_CN.md b/README_CN.md index 854bc76..768142c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -302,6 +302,12 @@ openclaw ltm search "咖啡" | `autoExperience` | 否 | LLM 自动提炼经验，默认 `true`。 | | `experienceRecall` | 否 | 召回结果是否包含经验，默认 `true`。 | | `inferOnAdd` | 否 | 写入时是否用 PowerMem 智能抽取，默认 `true`。 | +| `importMarkdownOnStart` | 否 | 启动时一次性导入已有 OpenClaw markdown 记忆，默认 `false`。 | +| `importMarkdownPaths` | 否 | 要导入的 markdown 文件或目录。默认扫描 `memory/`、`MEMORY.md`、`USER.md`；相对路径基于 OpenClaw workspace。 | +| `importMarkdownMaxFileBytes` | 否 | 单个 markdown 文件最大大小，默认 `10485760`（10 MiB）；超出的文件标记为 `skipped_too_large`。 | +| `importMarkdownBatchDelayMs` | 否 | 每个导入 chunk 之间的延迟，用于避免写入洪峰；默认 `300`。 | +| `importMarkdownMaxFiles` | 否 | 单次导入的 markdown 文件硬上限；不填表示不限制。 | +| `importMarkdownMaxChunks` | 否 | 单次导入的 markdown chunk 硬上限；不填表示不限制。 | | `dualWrite` | 否 | 仅 HTTP：远端 + 本地 SQLite 双写，远端失败自动排队补传。 | | `dualWritePriority` | 否 | 双写优先级：`"remote"`（默认）先远端 PowerMem、失败兜底本地 SQLite；`"local"` 先写/查 SQLite，再同步到远端。 | | `localDbPath` | 否 | 本地 SQLite 路径（`dualWrite`）。 | @@ -342,6 +348,8 @@ openclaw ltm search "咖啡" - `openclaw ltm search [--limit n]` — 搜索记忆 - `openclaw ltm health` — 检查 PowerMem 服务健康 - `openclaw ltm add ""` — 手动写入一条记忆 +- `openclaw ltm import-md [paths...] [--force] [--dry-run] [--delay-ms n] [--max-file-bytes n] [--max-files n] [--max-chunks n]` — 导入已有 markdown 记忆；不传路径时扫描 `memory/`、`MEMORY.md`、`USER.md` +- `openclaw ltm import-md-status [paths...] [--json]` — 查看每个 markdown 文件的导入状态：已导入、已变更、跳过、失败或未导入 --- diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 730c681..4c716d1 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -96,6 +96,41 @@ "label": "Infer on Add", "help": "Use PowerMem intelligent extraction when adding (infer=true)" }, + "importMarkdownOnStart": { + "label": "Import markdown on start", + "advanced": true, + "help": "One-time import of existing OpenClaw markdown memories on service start. Defaults to memory/, MEMORY.md, and USER.md." + }, + "importMarkdownPaths": { + "label": "Markdown import paths", + "advanced": true, + "placeholder": "[\"memory\", \"MEMORY.md\", \"USER.md\"]", + "help": "Files or directories to import when importMarkdownOnStart is enabled. Relative paths resolve from the OpenClaw workspace." + }, + "importMarkdownMaxFileBytes": { + "label": "Markdown import max file bytes", + "advanced": true, + "placeholder": "10485760", + "help": "Max size for a single markdown file. Files above this are marked skipped_too_large. Default: 10 MiB." + }, + "importMarkdownBatchDelayMs": { + "label": "Markdown import delay (ms)", + "advanced": true, + "placeholder": "300", + "help": "Delay between imported chunks to avoid startup write bursts. Default: 300ms." + }, + "importMarkdownMaxFiles": { + "label": "Markdown import max files", + "advanced": true, + "placeholder": "", + "help": "Optional hard cap on markdown files imported in one run. Empty means no cap." + }, + "importMarkdownMaxChunks": { + "label": "Markdown import max chunks", + "advanced": true, + "placeholder": "", + "help": "Optional hard cap on markdown chunks imported in one run. Empty means no cap." + }, "debugPerfLog": { "label": "Debug performance logs", "advanced": true, @@ -204,6 +239,15 @@ "autoExperience": { "type": "boolean" }, "experienceRecall": { "type": "boolean" }, "inferOnAdd": { "type": "boolean" }, + "importMarkdownOnStart": { "type": "boolean" }, + "importMarkdownPaths": { + "type": "array", + "items": { "type": "string" } + }, + "importMarkdownMaxFileBytes": { "type": "number" }, + "importMarkdownBatchDelayMs": { "type": "number" }, + "importMarkdownMaxFiles": { "type": "number" }, + "importMarkdownMaxChunks": { "type": "number" }, "debugPerfLog": { "type": "boolean" }, "perfSlowMs": { "type": "number" }, "dualWrite": { "type": "boolean" }, diff --git a/skills/install-memory-powermem-full/config-reference.md b/skills/install-memory-powermem-full/config-reference.md index cb3dba3..c38bf15 100644 --- a/skills/install-memory-powermem-full/config-reference.md +++ b/skills/install-memory-powermem-full/config-reference.md @@ -55,6 +55,12 @@ Quick reference for skill **`install-memory-powermem-full`**. See **SKILL.md** i | `autoExperience` | `true` | Auto-extract experiences via LLM. | | `experienceRecall` | `true` | Include experiences in recall. | | `inferOnAdd` | `true` | PowerMem intelligent extraction on add. | +| `importMarkdownOnStart` | `false` | One-time import of existing OpenClaw markdown memories on startup. | +| `importMarkdownPaths` | `memory`, `MEMORY.md`, `USER.md` | Markdown files/directories to import; relative paths resolve from the OpenClaw workspace. | +| `importMarkdownMaxFileBytes` | `10485760` | Max size for a single markdown file (10 MiB); larger files are marked `skipped_too_large`. | +| `importMarkdownBatchDelayMs` | `300` | Delay between imported chunks to avoid write bursts. | +| `importMarkdownMaxFiles` | — | Optional hard cap on markdown files imported in one run. | +| `importMarkdownMaxChunks` | — | Optional hard cap on markdown chunks imported in one run. | | `userId` | auto | Omit or set to `auto` to generate a stable ID saved under `/powermem/identity.json`. | | `agentId` | auto | Omit or set to `auto` to generate a stable ID saved under `/powermem/identity.json`. | | `dualWrite` | `false` | HTTP only: remote + local SQLite dual-write. | @@ -96,6 +102,8 @@ Quick reference for skill **`install-memory-powermem-full`**. See **SKILL.md** i openclaw ltm health openclaw ltm add "Something to remember" openclaw ltm search "query" +openclaw ltm import-md [paths...] [--force] [--dry-run] [--delay-ms n] [--max-file-bytes n] [--max-files n] [--max-chunks n] +openclaw ltm import-md-status [paths...] [--json] openclaw config set plugins.slots.memory none openclaw config set plugins.slots.memory memory-powermem diff --git a/src/config.ts b/src/config.ts index a578503..12153b6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -64,6 +64,18 @@ export type PowerMemConfig = { autoExperience: boolean; experienceRecall: boolean; inferOnAdd: boolean; + /** One-time import of existing OpenClaw markdown memories on service start. */ + importMarkdownOnStart?: boolean; + /** Files/directories to import, relative to the OpenClaw workspace unless absolute. */ + importMarkdownPaths?: string[]; + /** Max markdown file size to import. Default 10 MiB. */ + importMarkdownMaxFileBytes?: number; + /** Delay between imported chunks to avoid startup write bursts. Default 300ms. */ + importMarkdownBatchDelayMs?: number; + /** Optional hard cap on imported markdown files per run. Undefined = no cap. */ + importMarkdownMaxFiles?: number; + /** Optional hard cap on imported chunks per run. Undefined = no cap. */ + importMarkdownMaxChunks?: number; debugPerfLog?: boolean; perfSlowMs?: number; dualWrite?: boolean; @@ -111,6 +123,12 @@ const ALLOWED_KEYS = [ "autoExperience", "experienceRecall", "inferOnAdd", + "importMarkdownOnStart", + "importMarkdownPaths", + "importMarkdownMaxFileBytes", + "importMarkdownBatchDelayMs", + "importMarkdownMaxFiles", + "importMarkdownMaxChunks", "debugPerfLog", "perfSlowMs", "dualWrite", @@ -230,6 +248,18 @@ export const powerMemConfigSchema = { const syncBaseDelayMs = toPositiveInt(cfg.syncBaseDelayMs, 5000, 1000, 600000); const syncMaxDelayMs = toPositiveInt(cfg.syncMaxDelayMs, 60000, 1000, 3600000); const syncMaxRetries = toPositiveInt(cfg.syncMaxRetries, 10, 0, 100); + const importMarkdownBatchDelayMs = toPositiveInt( + cfg.importMarkdownBatchDelayMs, + 300, + 0, + 60000, + ); + const importMarkdownMaxFileBytes = toPositiveInt( + cfg.importMarkdownMaxFileBytes, + 10 * 1024 * 1024, + 1, + 1024 * 1024 * 1024, + ); return { mode, @@ -257,6 +287,12 @@ export const powerMemConfigSchema = { autoExperience: cfg.autoExperience !== false, experienceRecall: cfg.experienceRecall !== false, inferOnAdd: cfg.inferOnAdd !== false, + importMarkdownOnStart: cfg.importMarkdownOnStart === true, + importMarkdownPaths: parseStringList(cfg.importMarkdownPaths), + importMarkdownMaxFileBytes, + importMarkdownBatchDelayMs, + importMarkdownMaxFiles: toOptionalPositiveInt(cfg.importMarkdownMaxFiles, 1, 100000), + importMarkdownMaxChunks: toOptionalPositiveInt(cfg.importMarkdownMaxChunks, 1, 1000000), debugPerfLog: cfg.debugPerfLog === true, perfSlowMs: toPositiveInt(cfg.perfSlowMs, 800, 1, 600000), dualWrite: cfg.dualWrite === true, @@ -322,6 +358,24 @@ function toPositiveInt( return fallback; } +function toOptionalPositiveInt(v: unknown, min: number, max: number): number | undefined { + if (v === undefined || v === null || v === "") { + return undefined; + } + if (typeof v === "number" && Number.isFinite(v)) { + const n = Math.floor(v); + return n >= min ? Math.min(max, n) : undefined; + } + if (typeof v === "string" && v.trim() !== "") { + const n = Number(v); + if (Number.isFinite(n)) { + const floored = Math.floor(n); + return floored >= min ? Math.min(max, floored) : undefined; + } + } + return undefined; +} + function parseHeaderMap(value: unknown): Record | undefined { if (!value || typeof value !== "object" || Array.isArray(value)) { return undefined; @@ -335,6 +389,17 @@ function parseHeaderMap(value: unknown): Record | undefined { ) as Record; } +function parseStringList(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const items = value + .filter((item): item is string => typeof item === "string") + .map((item) => item.trim()) + .filter(Boolean); + return items.length > 0 ? items : undefined; +} + function pruneUndefined>(value: T): T | undefined { const entries = Object.entries(value).filter(([, v]) => v !== undefined); if (entries.length === 0) { @@ -373,6 +438,12 @@ export const DEFAULT_PLUGIN_CONFIG: PowerMemConfig = { autoExperience: true, experienceRecall: true, inferOnAdd: true, + importMarkdownOnStart: false, + importMarkdownPaths: undefined, + importMarkdownMaxFileBytes: 10 * 1024 * 1024, + importMarkdownBatchDelayMs: 300, + importMarkdownMaxFiles: undefined, + importMarkdownMaxChunks: undefined, debugPerfLog: false, perfSlowMs: 800, dualWrite: false, diff --git a/src/index.ts b/src/index.ts index e9b2f2c..ca5aa4d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -38,6 +38,12 @@ import { buildPowermemCliProcessEnv, } from "./openclaw-powermem-env.js"; import { resolvePmemExecutable } from "./resolve-powermem-cli.js"; +import { + buildMarkdownImportMarkerKey, + DEFAULT_MARKDOWN_IMPORT_PATHS, + getMarkdownImportStatus, + importMarkdownMemories, +} from "./markdown-import.js"; type GatewayApi = OpenClawPluginApi & { config?: unknown; @@ -434,6 +440,66 @@ const memoryPlugin = { if (cfg.dualWrite && "syncPending" in client) { void (client as DualWriteClient).syncPending("startup"); } + const markdownImportMarkerPath = join(stateDir, "powermem", "markdown-imports.json"); + const configuredMarkdownImportPaths = + cfg.importMarkdownPaths && cfg.importMarkdownPaths.length > 0 + ? cfg.importMarkdownPaths + : [...DEFAULT_MARKDOWN_IMPORT_PATHS]; + const runMarkdownImport = async (params: { + workspaceDir?: string; + paths?: readonly string[]; + force?: boolean; + dryRun?: boolean; + maxFileBytes?: number; + batchDelayMs?: number; + maxFiles?: number; + maxChunks?: number; + source: "startup" | "cli"; + }) => { + const paths = params.paths && params.paths.length > 0 + ? params.paths + : configuredMarkdownImportPaths; + const markerKey = buildMarkdownImportMarkerKey({ + userId, + agentId, + workspaceDir: params.workspaceDir, + paths, + }); + return importMarkdownMemories({ + client, + markerPath: markdownImportMarkerPath, + markerKey, + workspaceDir: params.workspaceDir, + paths, + infer: cfg.inferOnAdd, + force: params.force, + dryRun: params.dryRun, + maxFileBytes: params.maxFileBytes ?? cfg.importMarkdownMaxFileBytes, + batchDelayMs: params.batchDelayMs ?? cfg.importMarkdownBatchDelayMs, + maxFiles: params.maxFiles ?? cfg.importMarkdownMaxFiles, + maxChunks: params.maxChunks ?? cfg.importMarkdownMaxChunks, + source: params.source, + logger: api.logger, + }); + }; + const getMarkdownStatus = (params: { workspaceDir?: string; paths?: readonly string[] }) => { + const paths = params.paths && params.paths.length > 0 + ? params.paths + : configuredMarkdownImportPaths; + const markerKey = buildMarkdownImportMarkerKey({ + userId, + agentId, + workspaceDir: params.workspaceDir, + paths, + }); + return getMarkdownImportStatus({ + markerPath: markdownImportMarkerPath, + markerKey, + workspaceDir: params.workspaceDir, + paths, + maxFileBytes: cfg.importMarkdownMaxFileBytes, + }); + }; const resolvedPmem = cfg.mode === "cli" ? resolvePmemExecutable(cfg.pmemPath ?? DEFAULT_PMEM_PATH) : ""; const modeLabel = @@ -448,6 +514,12 @@ const memoryPlugin = { api.logger.info(`memory-powermem: perf logging enabled (slow >= ${perfSlowMs}ms)`); } + function parseOptionalCliInt(value: string | undefined): number | undefined { + if (!value || !value.trim()) return undefined; + const n = Number(value); + return Number.isFinite(n) && n >= 0 ? Math.floor(n) : undefined; + } + // ======================================================================== // Tools // ======================================================================== @@ -1399,6 +1471,99 @@ const memoryPlugin = { perfLog("cli.ltm.add.total", cliAddStartedAt, { textLen: text.trim().length }); } }); + + ltm + .command("import-md") + .description("Import existing OpenClaw markdown memory files") + .argument("[paths...]", "Markdown files or directories (default: memory/ MEMORY.md USER.md)") + .option("--force", "Import even if this workspace was already imported") + .option("--dry-run", "Scan and report files/chunks without writing memories") + .option("--delay-ms ", "Delay between imported chunks (default: config/importMarkdownBatchDelayMs)") + .option("--max-file-bytes ", "Max bytes for a single markdown file (default: config/importMarkdownMaxFileBytes)") + .option("--max-files ", "Max markdown files to import in this run") + .option("--max-chunks ", "Max chunks to import in this run") + .action(async (...args: unknown[]) => { + const cliImportStartedAt = perfNow(); + const maybePaths = Array.isArray(args[0]) ? (args[0] as string[]) : []; + const opts = (args[1] ?? {}) as { + force?: boolean; + dryRun?: boolean; + delayMs?: string; + maxFileBytes?: string; + maxFiles?: string; + maxChunks?: string; + }; + const delayMs = parseOptionalCliInt(opts.delayMs); + const maxFileBytes = parseOptionalCliInt(opts.maxFileBytes); + const maxFiles = parseOptionalCliInt(opts.maxFiles); + const maxChunks = parseOptionalCliInt(opts.maxChunks); + try { + const result = await runMarkdownImport({ + workspaceDir: process.cwd(), + paths: maybePaths, + force: opts.force === true, + dryRun: opts.dryRun === true, + maxFileBytes, + batchDelayMs: delayMs, + maxFiles, + maxChunks, + source: "cli", + }); + perfLog("cli.ltm.import_md", cliImportStartedAt, { + skipped: result.skipped, + reason: result.reason ?? null, + files: result.files, + chunks: result.chunks, + created: result.created, + limited: result.limited, + dryRun: opts.dryRun === true, + }); + if (result.skipped) { + console.log(`Markdown import skipped: ${result.reason}.`); + return; + } + console.log( + `Markdown import completed: files=${result.files}, chunks=${result.chunks}, stored=${result.created}${result.limited ? " (limited)" : ""}.`, + ); + } catch (err) { + console.error("Markdown import failed:", err); + process.exitCode = 1; + } + }); + + ltm + .command("import-md-status") + .description("Show markdown memory import status by file") + .argument("[paths...]", "Markdown files or directories (default: memory/ MEMORY.md USER.md)") + .option("--json", "Print machine-readable JSON") + .action((...args: unknown[]) => { + const maybePaths = Array.isArray(args[0]) ? (args[0] as string[]) : []; + const opts = (args[1] ?? {}) as { json?: boolean }; + const status = getMarkdownStatus({ + workspaceDir: process.cwd(), + paths: maybePaths, + }); + if (opts.json === true) { + console.log(JSON.stringify(status, null, 2)); + return; + } + console.log(`Markdown import marker: ${status.imported ? "found" : "not found"}`); + if (status.completedAt) { + console.log(`Completed at: ${status.completedAt}`); + } + console.log(`Workspace: ${status.workspaceDir}`); + if (status.files.length === 0) { + console.log("No markdown files found for the configured paths."); + return; + } + for (const file of status.files) { + const suffix = + file.error && file.status !== "skipped_too_large" ? ` (${file.error})` : ""; + console.log( + `${file.status}\tchunks=${file.chunks}\tcreated=${file.created}\t${file.path}${suffix}`, + ); + } + }); }, { commands: ["ltm"] }, ); @@ -1847,7 +2012,7 @@ const memoryPlugin = { api.registerService({ id: "memory-powermem", - start: async (_ctx: OpenClawPluginServiceContext) => { + start: async (ctx: OpenClawPluginServiceContext) => { try { const h = await client.health(); const where = @@ -1868,6 +2033,28 @@ const memoryPlugin = { `memory-powermem: health check failed (${hint}): ${String(err)}`, ); } + + if (cfg.importMarkdownOnStart) { + api.logger.info("memory-powermem: markdown import scheduled in background"); + void runMarkdownImport({ + workspaceDir: ctx.workspaceDir, + source: "startup", + }) + .then((result) => { + if (result.skipped) { + api.logger.info( + `memory-powermem: markdown import skipped (${result.reason ?? "unknown"})`, + ); + } else { + api.logger.info( + `memory-powermem: markdown import completed files=${result.files}, chunks=${result.chunks}, stored=${result.created}${result.limited ? " (limited)" : ""}`, + ); + } + }) + .catch((err) => { + api.logger.warn(`memory-powermem: markdown import failed: ${String(err)}`); + }); + } }, stop: (_ctx: OpenClawPluginServiceContext) => { api.logger.info("memory-powermem: stopped"); diff --git a/src/markdown-import.ts b/src/markdown-import.ts new file mode 100644 index 0000000..2b843aa --- /dev/null +++ b/src/markdown-import.ts @@ -0,0 +1,501 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; + +export const DEFAULT_MARKDOWN_IMPORT_PATHS = ["memory", "MEMORY.md", "USER.md"] as const; + +type MarkdownImportClient = { + add: ( + content: string, + options?: { infer?: boolean; metadata?: Record }, + ) => Promise>; +}; + +type Logger = { info?: (msg: string) => void; warn?: (msg: string) => void }; + +type MarkdownImportMarker = { + version: 1; + imports: Record< + string, + { + completedAt: string; + workspaceDir: string; + paths: string[]; + files: number; + chunks: number; + created: number; + limited?: boolean; + fileDetails?: MarkdownImportFileDetail[]; + } + >; +}; + +export type MarkdownImportFileStatus = + | "imported" + | "dry_run" + | "skipped_empty" + | "skipped_too_large" + | "read_failed" + | "limited"; + +export type MarkdownImportFileDetail = { + path: string; + size: number; + mtimeMs: number; + sha256?: string; + status: MarkdownImportFileStatus; + chunks: number; + created: number; + error?: string; +}; + +export type MarkdownImportResult = { + skipped: boolean; + reason?: string; + markerKey: string; + files: number; + chunks: number; + created: number; + paths: string[]; + limited: boolean; + fileDetails: MarkdownImportFileDetail[]; +}; + +export type MarkdownImportStatusEntry = { + path: string; + size: number; + mtimeMs: number; + status: + | MarkdownImportFileStatus + | "imported_changed" + | "not_imported"; + chunks: number; + created: number; + importedAt?: string; + error?: string; +}; + +export type MarkdownImportStatus = { + markerKey: string; + imported: boolean; + completedAt?: string; + workspaceDir: string; + paths: string[]; + files: MarkdownImportStatusEntry[]; +}; + +export type MarkdownImportOptions = { + client: MarkdownImportClient; + markerPath: string; + markerKey: string; + workspaceDir?: string; + paths?: readonly string[]; + infer: boolean; + force?: boolean; + dryRun?: boolean; + source: "startup" | "cli"; + maxFileBytes?: number; + maxChunkChars?: number; + maxFiles?: number; + maxChunks?: number; + batchDelayMs?: number; + logger?: Logger; +}; + +const DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024; +const DEFAULT_MAX_CHUNK_CHARS = 6000; + +export function buildMarkdownImportMarkerKey(params: { + userId: string; + agentId: string; + workspaceDir?: string; + paths: readonly string[]; +}): string { + const payload = JSON.stringify({ + userId: params.userId, + agentId: params.agentId, + workspaceDir: params.workspaceDir ? resolve(params.workspaceDir) : "", + paths: params.paths.map((p) => p.trim()).filter(Boolean), + }); + return createHash("sha256").update(payload).digest("hex").slice(0, 24); +} + +export async function importMarkdownMemories( + opts: MarkdownImportOptions, +): Promise { + const sourcePaths = normalizeSourcePaths(opts.paths); + const workspaceDir = opts.workspaceDir ? resolve(opts.workspaceDir) : process.cwd(); + const marker = loadMarker(opts.markerPath); + if (!opts.force && marker.imports[opts.markerKey]) { + return { + skipped: true, + reason: "already_imported", + markerKey: opts.markerKey, + files: 0, + chunks: 0, + created: 0, + paths: sourcePaths, + limited: false, + fileDetails: marker.imports[opts.markerKey].fileDetails ?? [], + }; + } + + const collectedFiles = collectMarkdownFiles(sourcePaths, workspaceDir, opts.maxFileBytes); + const fileDetails: MarkdownImportFileDetail[] = collectedFiles + .filter((file) => file.status === "skipped_too_large") + .map((file) => ({ + path: relative(workspaceDir, file.path) || file.path, + size: file.size, + mtimeMs: file.mtimeMs, + status: "skipped_too_large", + chunks: 0, + created: 0, + error: "file exceeds maxFileBytes", + })); + const eligibleFiles = collectedFiles.filter((file) => file.status === "eligible"); + const maxFiles = normalizePositiveLimit(opts.maxFiles); + const files = maxFiles === undefined ? eligibleFiles : eligibleFiles.slice(0, maxFiles); + if (files.length === 0) { + return { + skipped: true, + reason: "no_markdown_files", + markerKey: opts.markerKey, + files: 0, + chunks: 0, + created: 0, + paths: sourcePaths, + limited: false, + fileDetails, + }; + } + + const maxChunkChars = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS; + const maxChunks = normalizePositiveLimit(opts.maxChunks); + const batchDelayMs = normalizeDelayMs(opts.batchDelayMs); + const limitedByFiles = maxFiles !== undefined && eligibleFiles.length > files.length; + let limitedByChunks = false; + let chunkCount = 0; + let createdCount = 0; + const importedAt = new Date().toISOString(); + + if (limitedByFiles) { + for (const file of eligibleFiles.slice(files.length)) { + fileDetails.push(createFileDetail(file, workspaceDir, "limited", 0, 0)); + } + } + + for (const file of files) { + if (maxChunks !== undefined && chunkCount >= maxChunks) { + limitedByChunks = true; + fileDetails.push(createFileDetail(file, workspaceDir, "limited", 0, 0)); + break; + } + let content = ""; + try { + content = readFileSync(file.path, "utf-8").trim(); + } catch (err) { + opts.logger?.warn?.(`memory-powermem: markdown import skipped ${file.path}: ${String(err)}`); + fileDetails.push(createFileDetail(file, workspaceDir, "read_failed", 0, 0, String(err))); + continue; + } + if (!content) { + fileDetails.push(createFileDetail(file, workspaceDir, "skipped_empty", 0, 0)); + continue; + } + + const chunks = splitMarkdown(content, maxChunkChars); + const fileStartedChunkCount = chunkCount; + let fileCreatedCount = 0; + let fileLimited = false; + for (let i = 0; i < chunks.length; i++) { + if (maxChunks !== undefined && chunkCount >= maxChunks) { + limitedByChunks = true; + fileLimited = true; + break; + } + chunkCount += 1; + if (opts.dryRun) continue; + + const created = await opts.client.add(chunks[i], { + infer: opts.infer, + metadata: { + source: "markdown-import", + import_source: opts.source, + imported_at: importedAt, + file_path: relative(workspaceDir, file.path) || file.path, + chunk_index: i + 1, + chunk_total: chunks.length, + }, + }); + createdCount += created.length; + fileCreatedCount += created.length; + if (batchDelayMs > 0) { + await sleep(batchDelayMs); + } + } + fileDetails.push( + createFileDetail( + file, + workspaceDir, + fileLimited ? "limited" : opts.dryRun ? "dry_run" : "imported", + chunkCount - fileStartedChunkCount, + fileCreatedCount, + undefined, + opts.dryRun ? undefined : hashContent(content), + ), + ); + } + + if (!opts.dryRun) { + marker.imports[opts.markerKey] = { + completedAt: importedAt, + workspaceDir, + paths: sourcePaths, + files: files.length, + chunks: chunkCount, + created: createdCount, + limited: limitedByFiles || limitedByChunks, + fileDetails, + }; + saveMarker(opts.markerPath, marker); + } + + return { + skipped: false, + markerKey: opts.markerKey, + files: files.length, + chunks: chunkCount, + created: createdCount, + paths: sourcePaths, + limited: limitedByFiles || limitedByChunks, + fileDetails, + }; +} + +export function getMarkdownImportStatus(params: { + markerPath: string; + markerKey: string; + workspaceDir?: string; + paths?: readonly string[]; + maxFileBytes?: number; +}): MarkdownImportStatus { + const sourcePaths = normalizeSourcePaths(params.paths); + const workspaceDir = params.workspaceDir ? resolve(params.workspaceDir) : process.cwd(); + const marker = loadMarker(params.markerPath); + const batch = marker.imports[params.markerKey]; + const detailsByPath = new Map(); + for (const detail of batch?.fileDetails ?? []) { + detailsByPath.set(detail.path, detail); + } + const currentFiles = collectMarkdownFiles(sourcePaths, workspaceDir, params.maxFileBytes); + const files = currentFiles.map((file): MarkdownImportStatusEntry => { + const relPath = relative(workspaceDir, file.path) || file.path; + const previous = detailsByPath.get(relPath); + if (file.status === "skipped_too_large") { + return { + path: relPath, + size: file.size, + mtimeMs: file.mtimeMs, + status: "skipped_too_large", + chunks: previous?.chunks ?? 0, + created: previous?.created ?? 0, + importedAt: batch?.completedAt, + error: previous?.error ?? "file exceeds maxFileBytes", + }; + } + if (!previous) { + return { + path: relPath, + size: file.size, + mtimeMs: file.mtimeMs, + status: "not_imported", + chunks: 0, + created: 0, + }; + } + const changed = previous.size !== file.size || previous.mtimeMs !== file.mtimeMs; + return { + path: relPath, + size: file.size, + mtimeMs: file.mtimeMs, + status: changed && previous.status === "imported" ? "imported_changed" : previous.status, + chunks: previous.chunks, + created: previous.created, + importedAt: batch?.completedAt, + error: previous.error, + }; + }); + for (const previous of batch?.fileDetails ?? []) { + if (!files.some((file) => file.path === previous.path)) { + files.push({ + path: previous.path, + size: previous.size, + mtimeMs: previous.mtimeMs, + status: previous.status, + chunks: previous.chunks, + created: previous.created, + importedAt: batch?.completedAt, + error: "not found in current scan", + }); + } + } + return { + markerKey: params.markerKey, + imported: Boolean(batch), + completedAt: batch?.completedAt, + workspaceDir, + paths: sourcePaths, + files: files.sort((a, b) => a.path.localeCompare(b.path)), + }; +} + +function normalizeSourcePaths(paths: readonly string[] | undefined): string[] { + const normalized = (paths && paths.length > 0 ? paths : DEFAULT_MARKDOWN_IMPORT_PATHS) + .map((p) => p.trim()) + .filter(Boolean); + return normalized.length > 0 ? normalized : [...DEFAULT_MARKDOWN_IMPORT_PATHS]; +} + +type MarkdownFileCandidate = { + path: string; + size: number; + mtimeMs: number; + status: "eligible" | "skipped_too_large"; +}; + +function collectMarkdownFiles( + sourcePaths: readonly string[], + workspaceDir: string, + maxFileBytes = DEFAULT_MAX_FILE_BYTES, +): MarkdownFileCandidate[] { + const out: MarkdownFileCandidate[] = []; + const seen = new Set(); + for (const sourcePath of sourcePaths) { + const abs = resolveImportPath(sourcePath, workspaceDir); + collectOne(abs, out, seen, maxFileBytes); + } + return out.sort(); +} + +function collectOne( + absPath: string, + out: MarkdownFileCandidate[], + seen: Set, + maxFileBytes: number, +): void { + if (!existsSync(absPath)) return; + const st = statSync(absPath); + if (st.isFile()) { + if (absPath.toLowerCase().endsWith(".md") && !seen.has(absPath)) { + seen.add(absPath); + out.push({ + path: absPath, + size: st.size, + mtimeMs: st.mtimeMs, + status: st.size <= maxFileBytes ? "eligible" : "skipped_too_large", + }); + } + return; + } + if (!st.isDirectory()) return; + + for (const entry of readdirSync(absPath, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name === ".git") continue; + collectOne(join(absPath, entry.name), out, seen, maxFileBytes); + } +} + +function resolveImportPath(input: string, workspaceDir: string): string { + const expanded = input === "~" ? homedir() : input.replace(/^~(?=\/)/, homedir()); + return isAbsolute(expanded) ? resolve(expanded) : resolve(workspaceDir, expanded); +} + +function splitMarkdown(content: string, maxChunkChars: number): string[] { + if (content.length <= maxChunkChars) return [content]; + const paragraphs = content.split(/\n{2,}/); + const chunks: string[] = []; + let current = ""; + for (const paragraph of paragraphs) { + const next = current ? `${current}\n\n${paragraph}` : paragraph; + if (next.length <= maxChunkChars) { + current = next; + continue; + } + if (current) chunks.push(current); + if (paragraph.length <= maxChunkChars) { + current = paragraph; + continue; + } + for (let i = 0; i < paragraph.length; i += maxChunkChars) { + chunks.push(paragraph.slice(i, i + maxChunkChars)); + } + current = ""; + } + if (current) chunks.push(current); + return chunks.map((c) => c.trim()).filter(Boolean); +} + +function normalizePositiveLimit(value: number | undefined): number | undefined { + if (typeof value !== "number" || !Number.isFinite(value)) return undefined; + const n = Math.floor(value); + return n > 0 ? n : undefined; +} + +function normalizeDelayMs(value: number | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value)) return 0; + return Math.max(0, Math.min(60000, Math.floor(value))); +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +function createFileDetail( + file: MarkdownFileCandidate, + workspaceDir: string, + status: MarkdownImportFileStatus, + chunks: number, + created: number, + error?: string, + sha256?: string, +): MarkdownImportFileDetail { + return { + path: relative(workspaceDir, file.path) || file.path, + size: file.size, + mtimeMs: file.mtimeMs, + sha256, + status, + chunks, + created, + error, + }; +} + +function hashContent(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +function loadMarker(markerPath: string): MarkdownImportMarker { + try { + const parsed = JSON.parse(readFileSync(markerPath, "utf-8")); + if (parsed?.version === 1 && parsed.imports && typeof parsed.imports === "object") { + return parsed as MarkdownImportMarker; + } + } catch { + // Missing or invalid marker: start fresh. + } + return { version: 1, imports: {} }; +} + +function saveMarker(markerPath: string, marker: MarkdownImportMarker): void { + mkdirSync(dirname(markerPath), { recursive: true }); + writeFileSync(markerPath, JSON.stringify(marker, null, 2), "utf-8"); +} diff --git a/test/config.test.ts b/test/config.test.ts index 68d573f..9b938ab 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -70,6 +70,29 @@ describe("powerMemConfigSchema", () => { expect(DEFAULT_PLUGIN_CONFIG.pmemPath).toBe("bundled"); expect(DEFAULT_PLUGIN_CONFIG.useOpenClawModel).toBe(true); expect(DEFAULT_PLUGIN_CONFIG.dualWritePriority).toBe("remote"); + expect(DEFAULT_PLUGIN_CONFIG.importMarkdownOnStart).toBe(false); + expect(DEFAULT_PLUGIN_CONFIG.importMarkdownMaxFileBytes).toBe(10 * 1024 * 1024); + expect(DEFAULT_PLUGIN_CONFIG.importMarkdownBatchDelayMs).toBe(300); + expect(DEFAULT_PLUGIN_CONFIG.importMarkdownMaxFiles).toBeUndefined(); + expect(DEFAULT_PLUGIN_CONFIG.importMarkdownMaxChunks).toBeUndefined(); + }); + + it("parses markdown import config", () => { + const cfg = powerMemConfigSchema.parse({ + mode: "cli", + importMarkdownOnStart: true, + importMarkdownPaths: ["memory", "MEMORY.md", "", 123], + importMarkdownMaxFileBytes: "20971520", + importMarkdownBatchDelayMs: "250", + importMarkdownMaxFiles: "10", + importMarkdownMaxChunks: 20, + }) as PowerMemConfig; + expect(cfg.importMarkdownOnStart).toBe(true); + expect(cfg.importMarkdownPaths).toEqual(["memory", "MEMORY.md"]); + expect(cfg.importMarkdownMaxFileBytes).toBe(20 * 1024 * 1024); + expect(cfg.importMarkdownBatchDelayMs).toBe(250); + expect(cfg.importMarkdownMaxFiles).toBe(10); + expect(cfg.importMarkdownMaxChunks).toBe(20); }); it("parses dual-write local priority", () => { diff --git a/test/markdown-import.test.ts b/test/markdown-import.test.ts new file mode 100644 index 0000000..1dbf12a --- /dev/null +++ b/test/markdown-import.test.ts @@ -0,0 +1,149 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildMarkdownImportMarkerKey, + getMarkdownImportStatus, + importMarkdownMemories, +} from "../src/markdown-import.js"; + +describe("importMarkdownMemories", () => { + it("imports markdown files once and writes a marker", async () => { + const workspaceDir = mkdtempSync(join(tmpdir(), "pmem-md-workspace-")); + const stateDir = mkdtempSync(join(tmpdir(), "pmem-md-state-")); + writeFileSync(join(workspaceDir, "MEMORY.md"), "User prefers concise answers.", "utf-8"); + writeFileSync(join(workspaceDir, "notes.txt"), "not markdown", "utf-8"); + + const stored: string[] = []; + const markerKey = buildMarkdownImportMarkerKey({ + userId: "u", + agentId: "a", + workspaceDir, + paths: ["MEMORY.md"], + }); + const client = { + add: async (content: string) => { + stored.push(content); + return [{ memory_id: stored.length, content }]; + }, + }; + + const first = await importMarkdownMemories({ + client, + markerPath: join(stateDir, "markdown-imports.json"), + markerKey, + workspaceDir, + paths: ["MEMORY.md"], + infer: true, + source: "cli", + }); + expect(first.skipped).toBe(false); + expect(first.files).toBe(1); + expect(first.chunks).toBe(1); + expect(first.created).toBe(1); + expect(first.limited).toBe(false); + expect(first.fileDetails).toMatchObject([ + { + path: "MEMORY.md", + status: "imported", + chunks: 1, + created: 1, + }, + ]); + expect(stored).toEqual(["User prefers concise answers."]); + + const status = getMarkdownImportStatus({ + markerPath: join(stateDir, "markdown-imports.json"), + markerKey, + workspaceDir, + paths: ["MEMORY.md"], + }); + expect(status.imported).toBe(true); + expect(status.files).toMatchObject([ + { + path: "MEMORY.md", + status: "imported", + chunks: 1, + created: 1, + }, + ]); + + const second = await importMarkdownMemories({ + client, + markerPath: join(stateDir, "markdown-imports.json"), + markerKey, + workspaceDir, + paths: ["MEMORY.md"], + infer: true, + source: "cli", + }); + expect(second.skipped).toBe(true); + expect(second.reason).toBe("already_imported"); + expect(stored).toHaveLength(1); + }); + + it("respects max chunk limits", async () => { + const workspaceDir = mkdtempSync(join(tmpdir(), "pmem-md-workspace-")); + const stateDir = mkdtempSync(join(tmpdir(), "pmem-md-state-")); + writeFileSync(join(workspaceDir, "MEMORY.md"), "first paragraph\n\nsecond paragraph", "utf-8"); + + const stored: string[] = []; + const result = await importMarkdownMemories({ + client: { + add: async (content: string) => { + stored.push(content); + return [{ memory_id: stored.length, content }]; + }, + }, + markerPath: join(stateDir, "markdown-imports.json"), + markerKey: "limit-test", + workspaceDir, + paths: ["MEMORY.md"], + infer: true, + source: "cli", + maxChunkChars: 16, + maxChunks: 1, + }); + + expect(result.limited).toBe(true); + expect(result.chunks).toBe(1); + expect(result.fileDetails).toMatchObject([ + { + path: "MEMORY.md", + status: "limited", + chunks: 1, + }, + ]); + expect(stored).toHaveLength(1); + }); + + it("reports changed files after import", async () => { + const workspaceDir = mkdtempSync(join(tmpdir(), "pmem-md-workspace-")); + const stateDir = mkdtempSync(join(tmpdir(), "pmem-md-state-")); + const markerPath = join(stateDir, "markdown-imports.json"); + writeFileSync(join(workspaceDir, "MEMORY.md"), "before", "utf-8"); + + await importMarkdownMemories({ + client: { + add: async (content: string) => [{ memory_id: 1, content }], + }, + markerPath, + markerKey: "changed-test", + workspaceDir, + paths: ["MEMORY.md"], + infer: true, + source: "cli", + }); + + writeFileSync(join(workspaceDir, "MEMORY.md"), "after", "utf-8"); + const status = getMarkdownImportStatus({ + markerPath, + markerKey: "changed-test", + workspaceDir, + paths: ["MEMORY.md"], + }); + + expect(status.files[0].status).toBe("imported_changed"); + }); +}); From d338455af3a9444c50fa4ce3ba68f7a0a3107230 Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 6 May 2026 21:32:02 +0800 Subject: [PATCH 3/5] fixed: npm run lint --- src/better-sqlite3.d.ts | 29 +++++++++++++++++++++++++++++ src/llm.ts | 36 ++++++++++++++++++++---------------- src/openclaw-plugin-sdk.d.ts | 2 ++ 3 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 src/better-sqlite3.d.ts diff --git a/src/better-sqlite3.d.ts b/src/better-sqlite3.d.ts new file mode 100644 index 0000000..db34f57 --- /dev/null +++ b/src/better-sqlite3.d.ts @@ -0,0 +1,29 @@ +declare module "better-sqlite3" { + namespace Database { + interface RunResult { + changes: number; + lastInsertRowid: number | bigint; + } + + interface Statement { + run(...params: unknown[]): RunResult; + get(...params: unknown[]): any; + all(...params: unknown[]): any[]; + } + + interface Database { + exec(sql: string): this; + prepare(sql: string): Statement; + loadExtension(path: string): void; + close(): void; + } + } + + interface DatabaseConstructor { + new (filename: string, options?: Record): Database.Database; + (filename: string, options?: Record): Database.Database; + } + + const Database: DatabaseConstructor; + export = Database; +} diff --git a/src/llm.ts b/src/llm.ts index 9b82ff4..93542e4 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -3,9 +3,10 @@ import { getEnvApiKey, getModel, type Api, + type Message, type Model, } from "@mariozechner/pi-ai"; -import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core"; const API_REMAP: Record = { ollama: "openai-completions", @@ -20,14 +21,24 @@ function resolveCompatBaseUrl(originalApi: string, baseUrl: string | undefined): } type Logger = OpenClawPluginApi["logger"]; +type GatewayConfig = { + models?: { + providers?: Record; + }; + agents?: { + defaults?: { + model?: unknown; + }; + }; +}; function buildModelFromConfig( provider: string, modelId: string, - cfg: OpenClawPluginApi["config"], + cfg: unknown, logger: Logger, ): Model | null { - const providers = cfg?.models?.providers ?? {}; + const providers = (cfg as GatewayConfig | undefined)?.models?.providers ?? {}; const providerCfg = (providers as Record)[provider] ?? Object.entries(providers).find(([k]) => k.toLowerCase() === provider.toLowerCase())?.[1]; @@ -71,10 +82,10 @@ function buildModelFromConfig( function resolveModel( provider: string, modelId: string, - cfg: OpenClawPluginApi["config"], + cfg: unknown, logger: Logger, ): Model | null { - const builtIn = getModel(provider, modelId) as Model | null | undefined; + const builtIn = getModel(provider as any, modelId as any) as Model | null | undefined; if (builtIn) { logger.info?.(`powermem/llm: resolved model from built-in catalog — ${provider}/${modelId}`); return builtIn; @@ -109,7 +120,7 @@ async function resolveApiKey(api: OpenClawPluginApi, provider: string): Promise< // ignore } - const providers = (cfg?.models?.providers ?? {}) as Record>; + const providers = ((cfg as GatewayConfig | undefined)?.models?.providers ?? {}) as Record>; const providerCfg = providers[provider] ?? Object.values(providers).find( @@ -134,7 +145,7 @@ export async function callLlm( }, ): Promise { const cfg = api.config; - const defaultModel = cfg?.agents?.defaults?.model; + const defaultModel = (cfg as GatewayConfig | undefined)?.agents?.defaults?.model; const primary = typeof defaultModel === "string" ? defaultModel @@ -175,14 +186,7 @@ export async function callLlm( return null; } - const messages: Array<{ role: string; content: string; timestamp: number }> = []; - if (opts?.systemPrompt) { - messages.push({ - role: "system", - content: opts.systemPrompt, - timestamp: Date.now(), - }); - } + const messages: Message[] = []; messages.push({ role: "user", content: prompt, timestamp: Date.now() }); const maxTokens = opts?.maxTokens ?? 512; @@ -191,7 +195,7 @@ export async function callLlm( try { const result = await completeSimple( model, - { messages }, + { systemPrompt: opts?.systemPrompt, messages }, { apiKey, maxTokens, temperature }, ); const text = (result.content as Array<{ type: string; text?: string }>) diff --git a/src/openclaw-plugin-sdk.d.ts b/src/openclaw-plugin-sdk.d.ts index f7694d1..4c805b2 100644 --- a/src/openclaw-plugin-sdk.d.ts +++ b/src/openclaw-plugin-sdk.d.ts @@ -55,6 +55,8 @@ declare module "openclaw/plugin-sdk/memory-core" { } declare module "openclaw/plugin-sdk" { + export type { OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core"; + export type OpenClawPluginServiceContext = { config: unknown; workspaceDir?: string; From 3eeea08beb204199900e27e60a49660c54a39d86 Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 6 May 2026 23:05:47 +0800 Subject: [PATCH 4/5] add model_gateway --- src/config.ts | 1 + src/local-embedding.ts | 106 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/src/config.ts b/src/config.ts index 12153b6..8ea01fd 100644 --- a/src/config.ts +++ b/src/config.ts @@ -89,6 +89,7 @@ export type PowerMemConfig = { syncBaseDelayMs?: number; syncMaxDelayMs?: number; syncMaxRetries?: number; + /** Local sqlite-vec embeddings for dual-write. Use `provider: "model_gateway"` (alias `chj_gateway`) for the CHJ internal gateway: path `/embeddings/{model}`, header `X-CHJ-GWToken`; env fallbacks `MODEL_GATEWAY_BASE_URL`, `MODEL_GATEWAY_GW_TOKEN`. */ localVector?: { enabled?: boolean; provider?: string; diff --git a/src/local-embedding.ts b/src/local-embedding.ts index d2baa31..67cd3f2 100644 --- a/src/local-embedding.ts +++ b/src/local-embedding.ts @@ -1,3 +1,5 @@ +import { randomUUID } from "node:crypto"; + import { getEnvApiKey } from "@mariozechner/pi-ai"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core"; @@ -30,11 +32,14 @@ export type LocalEmbeddingFactory = { const DEFAULT_OPENAI_MODEL = "text-embedding-3-small"; const DEFAULT_OLLAMA_MODEL = "nomic-embed-text"; +/** Default embedding model id for `model_gateway` when none is configured. */ +const DEFAULT_MODEL_GATEWAY_MODEL = "qwen3-embedding-8b"; const PROVIDER_ALIASES: Record = { bailian: { provider: "openai", providerKey: "bailian" }, dashscope: { provider: "openai", providerKey: "dashscope" }, qwen: { provider: "openai", providerKey: "qwen" }, + chj_gateway: { provider: "model_gateway", providerKey: "model_gateway" }, }; function resolveProviderMapping(provider: string): { provider: string; providerKey: string } { @@ -53,11 +58,22 @@ function resolveDefaultModel(provider: string): string { if (provider === "ollama") { return DEFAULT_OLLAMA_MODEL; } + if (provider === "model_gateway") { + return DEFAULT_MODEL_GATEWAY_MODEL; + } return DEFAULT_OPENAI_MODEL; } function normalizeBaseUrl(provider: string, baseUrl?: string): string | undefined { const trimmed = baseUrl?.trim(); + if (provider === "model_gateway") { + const cleaned = trimmed?.replace(/\/+$/, ""); + if (cleaned) { + return cleaned; + } + const fromEnv = process.env.MODEL_GATEWAY_BASE_URL?.trim().replace(/\/+$/, ""); + return fromEnv || undefined; + } if (!trimmed) { if (provider === "ollama") { return "http://localhost:11434/v1"; @@ -75,6 +91,11 @@ function buildEmbeddingsUrl(baseUrl: string): string { return baseUrl.endsWith("/v1") ? `${baseUrl}/embeddings` : `${baseUrl}/v1/embeddings`; } +/** Path-style embeddings route used by the internal CHJ model gateway (PowerMem `ModelGatewayEmbedding`). */ +function buildModelGatewayEmbeddingsUrl(baseUrl: string, model: string): string { + return `${baseUrl.replace(/\/+$/, "")}/embeddings/${encodeURIComponent(model)}`; +} + /** Avoid `res.json()` on empty/truncated bodies (throws Unexpected end of JSON input). */ async function parseEmbeddingsResponseBody(res: Response): Promise { const text = await res.text(); @@ -164,9 +185,9 @@ function resolveLocalEmbeddingConfig(params: { ); const provider = resolvedProvider.provider; const providerKey = resolvedProvider.providerKey; - if (provider !== "openai" && provider !== "ollama") { + if (provider !== "openai" && provider !== "ollama" && provider !== "model_gateway") { params.logger.warn?.( - `dual-write: local vector search supports openai/ollama providers only, got "${provider}"`, + `dual-write: local vector search supports openai, ollama, and model_gateway providers, got "${provider}"`, ); return null; } @@ -255,11 +276,92 @@ async function resolveApiKey( return fallback; } +async function createModelGatewayEmbeddingProvider(params: { + api: OpenClawPluginApi; + cfg: LocalEmbeddingConfig; + logger: Logger; +}): Promise { + const providerKey = params.cfg.providerKey ?? params.cfg.provider; + const gwToken = + (await resolveApiKey( + params.api, + providerKey, + params.cfg.provider, + params.cfg.apiKey, + )) ?? + params.cfg.apiKey?.trim() ?? + process.env.MODEL_GATEWAY_GW_TOKEN?.trim() ?? + ""; + + const headers: Record = { + "Content-Type": "application/json", + ...(params.cfg.headers ?? {}), + }; + if (!headers["X-CHJ-GWToken"] && gwToken) { + headers["X-CHJ-GWToken"] = gwToken; + } + if (!headers["X-CHJ-GWToken"]) { + params.logger.warn?.( + "dual-write: model_gateway requires X-CHJ-GWToken (localVector.apiKey, MODEL_GATEWAY_GW_TOKEN, or headers)", + ); + return null; + } + + const baseUrl = params.cfg.baseUrl ?? normalizeBaseUrl("model_gateway"); + if (!baseUrl) { + params.logger.warn?.( + "dual-write: model_gateway requires baseUrl (localVector.baseUrl or MODEL_GATEWAY_BASE_URL)", + ); + return null; + } + + const url = buildModelGatewayEmbeddingsUrl(baseUrl, params.cfg.model); + + const embedBatch = async (texts: string[]): Promise => { + const res = await fetch(url, { + method: "POST", + headers: { + ...headers, + "BCS-APIHub-RequestId": randomUUID(), + }, + body: JSON.stringify({ input: texts }), + }); + const payload = (await parseEmbeddingsResponseBody(res)) as { + data?: Array<{ embedding?: number[] }>; + error?: { message?: string }; + }; + if (!res.ok) { + const message = + payload?.error?.message ?? + (typeof payload === "string" ? payload : `Embeddings failed: ${res.status}`); + throw new Error(message); + } + const rows = payload?.data ?? []; + return rows.map((row) => normalizeEmbedding(row.embedding ?? [])); + }; + + return { + id: params.cfg.provider, + model: params.cfg.model, + embed: async (text) => { + const single = text.replace(/\n/g, " "); + const [vec] = await embedBatch([single]); + return vec ?? []; + }, + embedBatch: async (texts) => + embedBatch(texts.map((t) => t.replace(/\n/g, " "))), + }; +} + async function createEmbeddingProvider(params: { api: OpenClawPluginApi; cfg: LocalEmbeddingConfig; logger: Logger; }): Promise { + if (params.cfg.provider === "model_gateway") { + return createModelGatewayEmbeddingProvider(params); + } + const providerKey = params.cfg.providerKey ?? params.cfg.provider; const apiKey = await resolveApiKey( params.api, From 4fb4f4e2c400fd413fcfd2fd37f455164df48322 Mon Sep 17 00:00:00 2001 From: Teingi Date: Wed, 6 May 2026 23:34:24 +0800 Subject: [PATCH 5/5] feat: load dual-write SQLite stack only for HTTP + dualWrite --- src/index.ts | 45 +++++++++++++++++++++--------------- src/openclaw-plugin-sdk.d.ts | 10 ++++++++ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/src/index.ts b/src/index.ts index ca5aa4d..5c39f3e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { Type } from "@sinclair/typebox"; import type { + OpenClawMemoryPlugin, OpenClawPluginApi, OpenClawPluginCliContext, } from "openclaw/plugin-sdk/memory-core"; @@ -28,9 +29,8 @@ import { dirname, join } from "node:path"; import { PowerMemClient, type PowerMemSearchResult } from "./client.js"; import { PowerMemV2Client } from "./client-v2.js"; import { PowerMemCLIClient } from "./client-cli.js"; -import { DualWriteClient } from "./dual-write-client.js"; +import type { DualWriteClient } from "./dual-write-client.js"; import { createLocalEmbeddingFactory } from "./local-embedding.js"; -import { LocalSqliteStore } from "./local-sqlite.js"; import { callLlm } from "./llm.js"; import { WalSession, walCapture as walCaptureCore } from "./wal.js"; import { @@ -219,7 +219,7 @@ const memoryPlugin = { kind: "memory" as const, configSchema: powerMemConfigSchema, - register(api: OpenClawPluginApi) { + async register(api: OpenClawPluginApi) { const gw = api as GatewayApi; const raw = api.pluginConfig; const toParse = @@ -287,19 +287,28 @@ const memoryPlugin = { agentId: cfg.localAgentId ?? identity.agentId, }); - const localStore = cfg.dualWrite - ? new LocalSqliteStore(resolveLocalDbPath(cfg, stateDir), { - logger: api.logger, - vector: { - enabled: cfg.localVector?.enabled ?? (cfg.dualWrite === true), - extensionPath: cfg.localVector?.extensionPath, - }, - }) - : null; + /** Dual-write SQLite + sqlite-vec load only when HTTP + dualWrite (avoids native better-sqlite3 at startup otherwise). */ + const needsDualWriteSqlite = cfg.dualWrite === true && cfg.mode === "http"; + + let DualWriteClientCtor: typeof import("./dual-write-client.js").DualWriteClient | undefined; + let localStore: InstanceType | null = null; + + if (needsDualWriteSqlite) { + const sqliteMod = await import("./local-sqlite.js"); + const dualMod = await import("./dual-write-client.js"); + DualWriteClientCtor = dualMod.DualWriteClient; + localStore = new sqliteMod.LocalSqliteStore(resolveLocalDbPath(cfg, stateDir), { + logger: api.logger, + vector: { + enabled: cfg.localVector?.enabled ?? true, + extensionPath: cfg.localVector?.extensionPath, + }, + }); + } const embeddingFactoryCache = new Map | null>(); const getLocalEmbeddingFactory = (ctxAgentId?: string) => { - if (!cfg.dualWrite) return null; + if (!needsDualWriteSqlite) return null; const key = ctxAgentId ?? "__default__"; if (embeddingFactoryCache.has(key)) { return embeddingFactoryCache.get(key) ?? null; @@ -332,10 +341,10 @@ const memoryPlugin = { } const httpClient = buildHttpClient(identity); - if (cfg.dualWrite && localStore) { + if (needsDualWriteSqlite && localStore && DualWriteClientCtor) { const localIdentity = resolveLocalIdentity(identity); const localEmbedding = getLocalEmbeddingFactory(ctxAgentId); - return new DualWriteClient(httpClient, localStore, { + return new DualWriteClientCtor(httpClient, localStore, { localUserId: localIdentity.userId, localAgentId: localIdentity.agentId, priority: cfg.dualWritePriority ?? "remote", @@ -437,7 +446,7 @@ const memoryPlugin = { } const client = getClientForAgent(); - if (cfg.dualWrite && "syncPending" in client) { + if (needsDualWriteSqlite && "syncPending" in client) { void (client as DualWriteClient).syncPending("startup"); } const markdownImportMarkerPath = join(stateDir, "powermem", "markdown-imports.json"); @@ -505,7 +514,7 @@ const memoryPlugin = { const modeLabel = cfg.mode === "cli" ? `cli (${resolvedPmem})` - : `${cfg.baseUrl}${cfg.httpApiVersion === "v2" ? " (v2)" : ""}${cfg.dualWrite ? " + sqlite" : ""}`; + : `${cfg.baseUrl}${cfg.httpApiVersion === "v2" ? " (v2)" : ""}${needsDualWriteSqlite ? " + sqlite" : ""}`; api.logger.info( `memory-powermem: plugin registered (mode: ${cfg.mode}, ${modeLabel}, user: ${userId}, agent: ${agentId})`, @@ -2061,6 +2070,6 @@ const memoryPlugin = { }, }); }, -}; +} satisfies OpenClawMemoryPlugin; export default memoryPlugin; diff --git a/src/openclaw-plugin-sdk.d.ts b/src/openclaw-plugin-sdk.d.ts index 4c805b2..510791a 100644 --- a/src/openclaw-plugin-sdk.d.ts +++ b/src/openclaw-plugin-sdk.d.ts @@ -52,6 +52,16 @@ declare module "openclaw/plugin-sdk/memory-core" { stop?: (ctx: ServiceContext) => void; }) => void; }; + + /** Memory plugin entry; `register` may be async when using dynamic imports (e.g. dual-write). */ + export type OpenClawMemoryPlugin = { + id: string; + name: string; + description: string; + kind: "memory"; + configSchema: unknown; + register: (api: OpenClawPluginApi) => void | Promise; + }; } declare module "openclaw/plugin-sdk" {