From 68cc1971cb9356c36c978ebea0633e4a2ae8deed Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 15:42:58 +0000 Subject: [PATCH] Add refresh-ebible-licenses command Adds a CLI command that detects the licenses of the eBible translations already in a database and writes them to the Translation.license column, so existing databases can pick up license information without re-downloading and re-importing every translation. eBible translations are found by their ebible.org details page URL (licenseUrl or website), which also covers translations that use override metadata. Translations whose license can't be detected are left unchanged. The optional --dir flag also writes the detected licenses into the matching metadata.json files so a later re-import doesn't clear them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01J6JNmbUF6TG2EarbCRrP3V --- package.json | 1 + packages/helloao-cli/README.md | 4 + packages/helloao-cli/actions.ts | 159 +++++++++++++++++++- packages/helloao-cli/cli.ts | 17 +++ packages/helloao-cli/ebible-license.spec.ts | 29 +++- packages/helloao-cli/ebible-license.ts | 28 ++++ 6 files changed, 236 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 59d2d07..2687937 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "fetch:bible:commentaries": "npm run cli fetch-tyndale-open-resources -- ./sources/comment/tyndale", "fetch:ebible": "npm run cli source-translations -- --convert-to-usx3 ./sources/ebible", "import:bible:ebible": "npm run cli import-translations -- --db ./free-use-bible-api.db ./sources/ebible", + "refresh:ebible:licenses": "npm run cli refresh-ebible-licenses -- --db ./free-use-bible-api.db --dir ./sources/ebible", "import:bible:metadata": "npm run cli import-bible-metadata -- --db ./free-use-bible-api.db ./sources/metadata", "import:bible:commentaries": "npm run cli import-commentaries -- --db ./free-use-bible-api.db ./sources/comment", "zip:sync": "npm run cli upload-api-files ./free-use-bible-api.zip", diff --git a/packages/helloao-cli/README.md b/packages/helloao-cli/README.md index 320aba3..7ab27c6 100644 --- a/packages/helloao-cli/README.md +++ b/packages/helloao-cli/README.md @@ -42,6 +42,8 @@ Commands: upload-api-files [options] Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder. source-translations [options] [translations...] Finds translation sources from ebible.org and downloads it. list-ebible-translations [search] List available eBible translations. Optionally filter by search term. + refresh-ebible-licenses [options] [translations...] Detects the licenses of the eBible translations that are already in the database and stores them. + Optionally limited to the given translation IDs. fetch-audio [options] [translations...] Fetches the specified audio translations and places them in the given directory. Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert" fetch-bible-metadata Fetches the Theographic bible metadata and places it in the given directory. @@ -49,6 +51,8 @@ Commands: help [command] display help for command ``` +> **eBible licenses.** `source-translations` detects the license of each eBible translation (from the Creative Commons link on its ebible.org details page, falling back to a "public domain" copyright notice) and stores it in `metadata.json`. To add licenses to a database that was imported before license detection existed, run `helloao refresh-ebible-licenses --db --dir `. It updates the `license` column of each eBible translation in place, and `--dir` also writes the licenses into the matching `metadata.json` files so that a later re-import doesn't clear them. Translations whose license can't be detected are left unchanged. + > **Audio timings.** The `import-audio-timings ` command loads per-verse start times for a chapter's audio into the database, from a JSON array of `{ translationId, bookId, chapterNumber, reader, verses }` records. To generate that file from the audio itself, see [`tools/audio-timings`](../../tools/audio-timings/README.md), which transcribes a chapter with whisperX and aligns the result against the chapter's known verse text. The `@helloao/cli` package can also be used as a library. diff --git a/packages/helloao-cli/actions.ts b/packages/helloao-cli/actions.ts index 630ae2b..5ee3301 100644 --- a/packages/helloao-cli/actions.ts +++ b/packages/helloao-cli/actions.ts @@ -14,6 +14,7 @@ import { bookOrder, dataset, InputTranslationMetadata, + License, } from '@helloao/tools/generation/index.js'; import * as theographic from '@helloao/tools/generation/theographic.js'; import fsExtra from 'fs-extra'; @@ -64,7 +65,7 @@ import { convertUsfmToUsx3, } from './conversion.js'; import { fetchEBibleMetadata } from './ebible.js'; -import { detectEBibleLicense } from './ebible-license.js'; +import { detectEBibleLicense, ebibleIdFromUrl } from './ebible-license.js'; import { importDatasetOutput } from './db.js'; import { groupBy } from 'es-toolkit/compat'; @@ -1485,6 +1486,162 @@ export async function listEBibleTranslations( } } +export interface RefreshEBibleLicensesOptions { + /** + * The database to update. + */ + db?: string | null; + + /** + * The directory that contains the sourced translations (e.g. the output directory of source-translations). + * If provided, the metadata.json files in this directory will also be updated with the detected licenses + * so that the licenses are preserved when the translations are re-imported. + */ + dir?: string; +} + +/** + * Detects the licenses of the eBible translations that are already in the database and stores them. + * This avoids having to re-download and re-import translations just to pick up license information. + * @param translations The IDs of the translations that should be refreshed. If empty, all eBible translations are refreshed. + * @param options The options. + */ +export async function refreshEBibleLicenses( + translations: string[], + options: RefreshEBibleLicensesOptions +): Promise { + const logger = log.getLogger(); + const db = await database.getDb(options.db); + try { + const rows = db + .prepare('SELECT id, licenseUrl, website FROM Translation;') + .all() as { id: string; licenseUrl: string; website: string }[]; + + const requested = new Set(translations); + const targets: { id: string; ebibleId: string }[] = []; + for (let row of rows) { + if (requested.size > 0 && !requested.has(row.id)) { + continue; + } + const ebibleId = + ebibleIdFromUrl(row.licenseUrl) ?? ebibleIdFromUrl(row.website); + if (ebibleId) { + targets.push({ id: row.id, ebibleId }); + } else if (requested.has(row.id)) { + logger.warn( + `${row.id} is not an eBible translation. Skipping.` + ); + } + } + + for (let id of requested) { + if (!rows.some((r) => r.id === id)) { + logger.warn(`${id} was not found in the database. Skipping.`); + } + } + + if (targets.length === 0) { + logger.log('No eBible translations found to refresh.'); + return; + } + + // The copyright text is used as a fallback when the details page has no Creative Commons license. + const copyrights = new Map(); + const storedSources = db + .prepare('SELECT id, copyright FROM EBibleSource;') + .all() as { id: string; copyright: string }[]; + for (let source of storedSources) { + copyrights.set(source.id.toLowerCase(), source.copyright); + } + try { + for (let source of await fetchEBibleMetadata()) { + copyrights.set(source.id.toLowerCase(), source.copyright); + } + } catch (err) { + logger.warn( + 'Failed to fetch the eBible translation list. Using the copyright text stored in the database.', + err + ); + } + + logger.log(`Refreshing licenses for ${targets.length} translations...`); + + const updateLicense = db.prepare( + 'UPDATE Translation SET license = @license WHERE id = @id;' + ); + const licenses = new Map(); + for (let [index, target] of targets.entries()) { + const progress = `[${index + 1}/${targets.length}]`; + const license = await detectEBibleLicense({ + id: target.ebibleId, + copyright: copyrights.get(target.ebibleId.toLowerCase()) ?? '', + }); + + if (!license) { + logger.log( + `${progress} ${target.id}: No license detected. Leaving unchanged.` + ); + continue; + } + + updateLicense.run({ + id: target.id, + license: JSON.stringify(license), + }); + licenses.set(target.id, license); + logger.log(`${progress} ${target.id}: ${JSON.stringify(license)}`); + } + + logger.log( + `Updated licenses for ${licenses.size} of ${targets.length} translations.` + ); + + if (options.dir) { + const updatedFiles = await updateMetadataLicenses( + options.dir, + licenses + ); + logger.log( + `Updated ${updatedFiles} metadata.json files in ${options.dir}.` + ); + } + } finally { + db.close(); + } +} + +/** + * Sets the license in the metadata.json file of each translation in the given directory. + * @param dir The directory that contains the translation directories. + * @param licenses The map of translation IDs to their licenses. + * @returns The number of metadata.json files that were updated. + */ +async function updateMetadataLicenses( + dir: string, + licenses: Map +): Promise { + let updated = 0; + const entries = await readdir(dir, { withFileTypes: true }); + for (let entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const metadataPath = path.resolve(dir, entry.name, 'metadata.json'); + if (!(await exists(metadataPath))) { + continue; + } + const metadata = JSON.parse(await readFile(metadataPath, 'utf-8')); + const license = licenses.get(metadata.id); + if (!license) { + continue; + } + metadata.license = license; + await writeFile(metadataPath, JSON.stringify(metadata, null, 2)); + updated++; + } + return updated; +} + /** * Generates the translation files directly from the translation stored in the given input directory. * @param input The input directory that the translation is stored in. diff --git a/packages/helloao-cli/cli.ts b/packages/helloao-cli/cli.ts index 8a4d1ba..47075b5 100644 --- a/packages/helloao-cli/cli.ts +++ b/packages/helloao-cli/cli.ts @@ -25,6 +25,7 @@ import { importTranslations, initDb, listEBibleTranslations, + refreshEBibleLicenses, sourceTranslations, searchTypesenseVerses, uploadTestTranslation, @@ -902,6 +903,22 @@ async function start() { await listEBibleTranslations(search); }); + program + .command('refresh-ebible-licenses [translations...]') + .description( + 'Detects the licenses of the eBible translations that are already in the database and stores them.\nOptionally limited to the given translation IDs.' + ) + .option( + '--dir ', + 'The directory that the translations were sourced into. If provided, their metadata.json files are also updated so that the licenses are kept when they are re-imported.' + ) + .action(async (translations: string[], options: any) => { + await refreshEBibleLicenses(translations, { + ...program.opts(), + ...options, + }); + }); + program .command('fetch-audio [translations...]') .description( diff --git a/packages/helloao-cli/ebible-license.spec.ts b/packages/helloao-cli/ebible-license.spec.ts index fc53a06..978dae6 100644 --- a/packages/helloao-cli/ebible-license.spec.ts +++ b/packages/helloao-cli/ebible-license.spec.ts @@ -3,6 +3,7 @@ import { extractCcLicenseFromDetailsHtml, isPublicDomainCopyrightText, detectEBibleLicense, + ebibleIdFromUrl, } from './ebible-license'; describe('ccLicenseFromUrl()', () => { @@ -154,6 +155,30 @@ describe('extractCcLicenseFromDetailsHtml()', () => { }); }); +describe('ebibleIdFromUrl()', () => { + const cases = [ + ['https://ebible.org/Scriptures/details.php?id=engwebp', 'engwebp'], + ['https://ebible.org/Scriptures/details.php?id=engDRA', 'engDRA'], + ['http://www.ebible.org/Scriptures/details.php?id=hin2017', 'hin2017'], + [ + 'https://EBIBLE.ORG/scriptures/DETAILS.php?foo=bar&id=arbnav', + 'arbnav', + ], + ['https://ebible.org/Scriptures/details.php', null], + ['https://ebible.org/Scriptures/details.php?id=', null], + ['https://ebible.org/find/details.php?id=engwebp', null], + ['https://example.com/Scriptures/details.php?id=engwebp', null], + ['https://berean.bible/', null], + ['not a url at all', null], + [null, null], + [undefined, null], + ] as const; + + it.each(cases)('should map %s to %s', (url, expected) => { + expect(ebibleIdFromUrl(url)).toBe(expected); + }); +}); + describe('isPublicDomainCopyrightText()', () => { const cases = [ ['This work has been dedicated to the Public Domain.', true], @@ -220,7 +245,9 @@ describe('detectEBibleLicense()', () => { }); it('should fall back to the copyright text when the fetch fails', async () => { - global.fetch = jest.fn().mockRejectedValue(new Error('network error')) as any; + global.fetch = jest + .fn() + .mockRejectedValue(new Error('network error')) as any; const license = await detectEBibleLicense({ id: 'engwmb', diff --git a/packages/helloao-cli/ebible-license.ts b/packages/helloao-cli/ebible-license.ts index ad63558..4cd99b5 100644 --- a/packages/helloao-cli/ebible-license.ts +++ b/packages/helloao-cli/ebible-license.ts @@ -109,6 +109,34 @@ export function extractCcLicenseFromDetailsHtml(html: string): License | null { return null; } +/** + * Gets the eBible translation ID from an ebible.org translation details page URL + * (e.g. `https://ebible.org/Scriptures/details.php?id=engwebp` returns `engwebp`). + * Returns null if the URL isn't an ebible.org details page URL. + */ +export function ebibleIdFromUrl(url: string | null | undefined): string | null { + if (!url) { + return null; + } + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + + const host = parsed.hostname.toLowerCase(); + if (host !== 'ebible.org' && host !== 'www.ebible.org') { + return null; + } + if (parsed.pathname.toLowerCase() !== '/scriptures/details.php') { + return null; + } + + return parsed.searchParams.get('id') || null; +} + /** * Determines whether the given ebible.org copyright notice indicates that * the translation has been dedicated to the public domain.