From 4f8136565ae2053af05bc2fe67e1a4b1aff23b4a Mon Sep 17 00:00:00 2001 From: 3446798488 <3446798488@qq.com> Date: Tue, 4 Jun 2024 04:03:56 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20i18n=20Markdown?= =?UTF-8?q?=20content=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/lobe-i18n/package.json | 1 + packages/lobe-i18n/src/cache/index.ts | 37 +++++++++++++++++++ .../src/commands/TranslateMarkdown/index.tsx | 22 +++++++---- packages/lobe-i18n/src/core/I18n.ts | 2 + 4 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 packages/lobe-i18n/src/cache/index.ts diff --git a/packages/lobe-i18n/package.json b/packages/lobe-i18n/package.json index c85d8c9..cdfee5c 100644 --- a/packages/lobe-i18n/package.json +++ b/packages/lobe-i18n/package.json @@ -54,6 +54,7 @@ "type-check": "tsc --noEmit" }, "dependencies": { + "@deno/kv": "^0.7.0", "@inkjs/ui": "^1", "@langchain/core": "latest", "@langchain/openai": "latest", diff --git a/packages/lobe-i18n/src/cache/index.ts b/packages/lobe-i18n/src/cache/index.ts new file mode 100644 index 0000000..d02be10 --- /dev/null +++ b/packages/lobe-i18n/src/cache/index.ts @@ -0,0 +1,37 @@ +import { Kv, openKv } from '@deno/kv'; +import crypto from 'node:crypto'; + +export class FileHashCache { + kv: Promise; + constructor(cacheDB = '.lobe-i18n.db') { + this.kv = openKv(cacheDB); + } + getHash(fileContent: string) { + return crypto.createHash('md5').update(fileContent).digest('hex'); + } + setHash(filePath: string, key: string) { + return this.kv.then((kv) => { + return kv.set(['files', filePath], key); + }); + } + hasHash(filePath: string, key: string) { + return this.kv + .then((kv) => { + return kv.get(['files', filePath]); + }) + .then((res) => { + return res.value === key; + }); + } + remove(filePath: string) { + return this.kv.then((kv) => kv.delete(['files', filePath])); + } + clear() { + return this.kv.then((kv) => { + return kv.delete(['files']); + }); + } + destory() { + return this.kv.then((kv) => kv.close()); + } +} diff --git a/packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx b/packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx index 549a652..7f448c1 100644 --- a/packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx +++ b/packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx @@ -4,9 +4,9 @@ import { consola } from 'consola'; import { globSync } from 'glob'; import matter from 'gray-matter'; import { isString } from 'lodash-es'; -import { existsSync } from 'node:fs'; import { relative, resolve } from 'node:path'; +import { FileHashCache } from '@/cache'; import Progress from '@/components/Progress'; import { I18n, I18nMarkdownWriteOptions } from '@/core/I18n'; import { selectors } from '@/store'; @@ -21,6 +21,7 @@ class TranslateMarkdown { markdownConfig: MarkdownConfig; query: I18nMarkdownWriteOptions[] = []; i18n: I18n; + fileHashCache = new FileHashCache(); constructor() { this.markdownConfig = selectors.getMarkdownConfigFile(); const defaultConfig = selectors.getConfigFile(); @@ -55,7 +56,7 @@ class TranslateMarkdown { if (!files || files.length === 0) alert.error('No markdown entry was found.', true); - this.genFilesQuery(files); + await this.genFilesQuery(files); if (this.query.length > 0) { await this.runQuery(); @@ -64,7 +65,6 @@ class TranslateMarkdown { } consola.success('All i18n tasks have been completed！'); } - async runQuery() { consola.info( `Current model setting: ${chalk.cyan(this.config.modelName)} (temperature: ${chalk.cyan( @@ -101,6 +101,7 @@ class TranslateMarkdown { } writeMarkdown(item.filename, mdResut); + await this.fileHashCache.setHash(item.filePath, item.hash!); totalTokenUsage += data.tokenUsage; consola.success(chalk.yellow(outputPath), chalk.gray(`[Token usage: ${data.tokenUsage}]`)); } else { @@ -110,7 +111,7 @@ class TranslateMarkdown { if (totalTokenUsage > 0) consola.info('Total token usage:', chalk.cyan(totalTokenUsage)); } - genFilesQuery(files: string[], skipLog?: boolean) { + async genFilesQuery(files: string[], skipLog?: boolean) { const config = this.markdownConfig; if (!skipLog) consola.start( @@ -121,24 +122,31 @@ class TranslateMarkdown { for (const file of files) { try { const md = readMarkdown(file); + const hash = this.fileHashCache.getHash(md); + const cacheKey = await this.fileHashCache.hasHash(file, hash); + if (cacheKey) { + consola.info(`✅ Cahced: ${chalk.green(file)}`); + continue; + } for (const locale of config.outputLocales || []) { const targetExtension = this.getTargetExtension(locale, file, md); const targetFilename = this.getTargetFilename(file, targetExtension); - if (existsSync(targetFilename)) continue; const mode = this.getMode(file, md); const { data, content } = matter(md); consola.info(`📄 To ${locale}: ${chalk.yellow(targetFilename)}`); this.query.push({ + filePath: file, filename: targetFilename, from: config.entryLocale, + hash, matter: data, md: this.markdownConfig.includeMatter ? md : content, mode, to: locale, }); } - } catch { - alert.error(`${file} not found`, true); + } catch (error: unknown) { + alert.error(`${file} not found. ` + (error as Error).message, true); } } } diff --git a/packages/lobe-i18n/src/core/I18n.ts b/packages/lobe-i18n/src/core/I18n.ts index a050b1e..63c4ace 100644 --- a/packages/lobe-i18n/src/core/I18n.ts +++ b/packages/lobe-i18n/src/core/I18n.ts @@ -33,6 +33,8 @@ export interface I18nTranslateOptions { export interface I18nMarkdownTranslateOptions extends Pick { + filePath: string; + hash: string; matter?: any; md: string; mode: MarkdownModeType; From 38a7598f293c9d429697a81962181604db246ab1 Mon Sep 17 00:00:00 2001 From: 3446798488 <3446798488@qq.com> Date: Tue, 4 Jun 2024 06:00:42 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20cli=20option=20to?= =?UTF-8?q?=20clear=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/lobe-i18n/src/cache/index.ts | 5 +++-- packages/lobe-i18n/src/cli.tsx | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/lobe-i18n/src/cache/index.ts b/packages/lobe-i18n/src/cache/index.ts index d02be10..7abf734 100644 --- a/packages/lobe-i18n/src/cache/index.ts +++ b/packages/lobe-i18n/src/cache/index.ts @@ -27,8 +27,9 @@ export class FileHashCache { return this.kv.then((kv) => kv.delete(['files', filePath])); } clear() { - return this.kv.then((kv) => { - return kv.delete(['files']); + return this.kv.then(async (kv) => { + const iter = kv.list({ prefix: ['files'] }); + for await (const res of iter) kv.delete(res.key); }); } destory() { diff --git a/packages/lobe-i18n/src/cli.tsx b/packages/lobe-i18n/src/cli.tsx index 7123d9c..9529cc4 100644 --- a/packages/lobe-i18n/src/cli.tsx +++ b/packages/lobe-i18n/src/cli.tsx @@ -16,6 +16,7 @@ notifier.notify({ isGlobal: true }); const program = new Command(); interface Flags { + clear?: boolean; config?: string; option?: boolean; withMd?: boolean; @@ -27,6 +28,7 @@ program .version(packageJson.version) .addOption(new Option('-o, --option', 'Setup lobe-i18n preferences')) .addOption(new Option('-c, --config ', 'Specify the configuration file')) + .addOption(new Option('--clear', 'clear cache')) .addOption( new Option('-m, --with-md', 'Run i18n translation and markdown translation simultaneously'), ); @@ -45,7 +47,11 @@ program.command('locale', { isDefault: true }).action(async () => { program.command('md').action(async () => { const options: Flags = program.opts(); if (options.config) explorer.loadCustomConfig(options.config); - await new TranslateMarkdown().start(); + const service = new TranslateMarkdown(); + if (options.clear) { + await service.fileHashCache.clear(); + } + await service.start(); }); program.parse();