diff --git a/package.json b/package.json index 59d2d07..2687937 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "fetch:bible:commentaries": "npm run cli fetch-tyndale-open-resources -- ./sources/comment/tyndale", "fetch:ebible": "npm run cli source-translations -- --convert-to-usx3 ./sources/ebible", "import:bible:ebible": "npm run cli import-translations -- --db ./free-use-bible-api.db ./sources/ebible", + "refresh:ebible:licenses": "npm run cli refresh-ebible-licenses -- --db ./free-use-bible-api.db --dir ./sources/ebible", "import:bible:metadata": "npm run cli import-bible-metadata -- --db ./free-use-bible-api.db ./sources/metadata", "import:bible:commentaries": "npm run cli import-commentaries -- --db ./free-use-bible-api.db ./sources/comment", "zip:sync": "npm run cli upload-api-files ./free-use-bible-api.zip", diff --git a/packages/helloao-cli/README.md b/packages/helloao-cli/README.md index 320aba3..7ab27c6 100644 --- a/packages/helloao-cli/README.md +++ b/packages/helloao-cli/README.md @@ -42,6 +42,8 @@ Commands: upload-api-files [options] Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder. source-translations [options] [translations...] Finds translation sources from ebible.org and downloads it. list-ebible-translations [search] List available eBible translations. Optionally filter by search term. + refresh-ebible-licenses [options] [translations...] Detects the licenses of the eBible translations that are already in the database and stores them. + Optionally limited to the given translation IDs. fetch-audio [options] [translations...] Fetches the specified audio translations and places them in the given directory. Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert" fetch-bible-metadata Fetches the Theographic bible metadata and places it in the given directory. @@ -49,6 +51,8 @@ Commands: help [command] display help for command ``` +> **eBible licenses.** `source-translations` detects the license of each eBible translation (from the Creative Commons link on its ebible.org details page, falling back to a "public domain" copyright notice) and stores it in `metadata.json`. To add licenses to a database that was imported before license detection existed, run `helloao refresh-ebible-licenses --db --dir `. It updates the `license` column of each eBible translation in place, and `--dir` also writes the licenses into the matching `metadata.json` files so that a later re-import doesn't clear them. Translations whose license can't be detected are left unchanged. + > **Audio timings.** The `import-audio-timings ` command loads per-verse start times for a chapter's audio into the database, from a JSON array of `{ translationId, bookId, chapterNumber, reader, verses }` records. To generate that file from the audio itself, see [`tools/audio-timings`](../../tools/audio-timings/README.md), which transcribes a chapter with whisperX and aligns the result against the chapter's known verse text. The `@helloao/cli` package can also be used as a library. diff --git a/packages/helloao-cli/actions.ts b/packages/helloao-cli/actions.ts index 630ae2b..5ee3301 100644 --- a/packages/helloao-cli/actions.ts +++ b/packages/helloao-cli/actions.ts @@ -14,6 +14,7 @@ import { bookOrder, dataset, InputTranslationMetadata, + License, } from '@helloao/tools/generation/index.js'; import * as theographic from '@helloao/tools/generation/theographic.js'; import fsExtra from 'fs-extra'; @@ -64,7 +65,7 @@ import { convertUsfmToUsx3, } from './conversion.js'; import { fetchEBibleMetadata } from './ebible.js'; -import { detectEBibleLicense } from './ebible-license.js'; +import { detectEBibleLicense, ebibleIdFromUrl } from './ebible-license.js'; import { importDatasetOutput } from './db.js'; import { groupBy } from 'es-toolkit/compat'; @@ -1485,6 +1486,162 @@ export async function listEBibleTranslations( } } +export interface RefreshEBibleLicensesOptions { + /** + * The database to update. + */ + db?: string | null; + + /** + * The directory that contains the sourced translations (e.g. the output directory of source-translations). + * If provided, the metadata.json files in this directory will also be updated with the detected licenses + * so that the licenses are preserved when the translations are re-imported. + */ + dir?: string; +} + +/** + * Detects the licenses of the eBible translations that are already in the database and stores them. + * This avoids having to re-download and re-import translations just to pick up license information. + * @param translations The IDs of the translations that should be refreshed. If empty, all eBible translations are refreshed. + * @param options The options. + */ +export async function refreshEBibleLicenses( + translations: string[], + options: RefreshEBibleLicensesOptions +): Promise { + const logger = log.getLogger(); + const db = await database.getDb(options.db); + try { + const rows = db + .prepare('SELECT id, licenseUrl, website FROM Translation;') + .all() as { id: string; licenseUrl: string; website: string }[]; + + const requested = new Set(translations); + const targets: { id: string; ebibleId: string }[] = []; + for (let row of rows) { + if (requested.size > 0 && !requested.has(row.id)) { + continue; + } + const ebibleId = + ebibleIdFromUrl(row.licenseUrl) ?? ebibleIdFromUrl(row.website); + if (ebibleId) { + targets.push({ id: row.id, ebibleId }); + } else if (requested.has(row.id)) { + logger.warn( + `${row.id} is not an eBible translation. Skipping.` + ); + } + } + + for (let id of requested) { + if (!rows.some((r) => r.id === id)) { + logger.warn(`${id} was not found in the database. Skipping.`); + } + } + + if (targets.length === 0) { + logger.log('No eBible translations found to refresh.'); + return; + } + + // The copyright text is used as a fallback when the details page has no Creative Commons license. + const copyrights = new Map(); + const storedSources = db + .prepare('SELECT id, copyright FROM EBibleSource;') + .all() as { id: string; copyright: string }[]; + for (let source of storedSources) { + copyrights.set(source.id.toLowerCase(), source.copyright); + } + try { + for (let source of await fetchEBibleMetadata()) { + copyrights.set(source.id.toLowerCase(), source.copyright); + } + } catch (err) { + logger.warn( + 'Failed to fetch the eBible translation list. Using the copyright text stored in the database.', + err + ); + } + + logger.log(`Refreshing licenses for ${targets.length} translations...`); + + const updateLicense = db.prepare( + 'UPDATE Translation SET license = @license WHERE id = @id;' + ); + const licenses = new Map(); + for (let [index, target] of targets.entries()) { + const progress = `[${index + 1}/${targets.length}]`; + const license = await detectEBibleLicense({ + id: target.ebibleId, + copyright: copyrights.get(target.ebibleId.toLowerCase()) ?? '', + }); + + if (!license) { + logger.log( + `${progress} ${target.id}: No license detected. Leaving unchanged.` + ); + continue; + } + + updateLicense.run({ + id: target.id, + license: JSON.stringify(license), + }); + licenses.set(target.id, license); + logger.log(`${progress} ${target.id}: ${JSON.stringify(license)}`); + } + + logger.log( + `Updated licenses for ${licenses.size} of ${targets.length} translations.` + ); + + if (options.dir) { + const updatedFiles = await updateMetadataLicenses( + options.dir, + licenses + ); + logger.log( + `Updated ${updatedFiles} metadata.json files in ${options.dir}.` + ); + } + } finally { + db.close(); + } +} + +/** + * Sets the license in the metadata.json file of each translation in the given directory. + * @param dir The directory that contains the translation directories. + * @param licenses The map of translation IDs to their licenses. + * @returns The number of metadata.json files that were updated. + */ +async function updateMetadataLicenses( + dir: string, + licenses: Map +): Promise { + let updated = 0; + const entries = await readdir(dir, { withFileTypes: true }); + for (let entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const metadataPath = path.resolve(dir, entry.name, 'metadata.json'); + if (!(await exists(metadataPath))) { + continue; + } + const metadata = JSON.parse(await readFile(metadataPath, 'utf-8')); + const license = licenses.get(metadata.id); + if (!license) { + continue; + } + metadata.license = license; + await writeFile(metadataPath, JSON.stringify(metadata, null, 2)); + updated++; + } + return updated; +} + /** * Generates the translation files directly from the translation stored in the given input directory. * @param input The input directory that the translation is stored in. diff --git a/packages/helloao-cli/cli.ts b/packages/helloao-cli/cli.ts index 8a4d1ba..47075b5 100644 --- a/packages/helloao-cli/cli.ts +++ b/packages/helloao-cli/cli.ts @@ -25,6 +25,7 @@ import { importTranslations, initDb, listEBibleTranslations, + refreshEBibleLicenses, sourceTranslations, searchTypesenseVerses, uploadTestTranslation, @@ -902,6 +903,22 @@ async function start() { await listEBibleTranslations(search); }); + program + .command('refresh-ebible-licenses [translations...]') + .description( + 'Detects the licenses of the eBible translations that are already in the database and stores them.\nOptionally limited to the given translation IDs.' + ) + .option( + '--dir ', + 'The directory that the translations were sourced into. If provided, their metadata.json files are also updated so that the licenses are kept when they are re-imported.' + ) + .action(async (translations: string[], options: any) => { + await refreshEBibleLicenses(translations, { + ...program.opts(), + ...options, + }); + }); + program .command('fetch-audio [translations...]') .description( diff --git a/packages/helloao-cli/ebible-license.spec.ts b/packages/helloao-cli/ebible-license.spec.ts index fc53a06..978dae6 100644 --- a/packages/helloao-cli/ebible-license.spec.ts +++ b/packages/helloao-cli/ebible-license.spec.ts @@ -3,6 +3,7 @@ import { extractCcLicenseFromDetailsHtml, isPublicDomainCopyrightText, detectEBibleLicense, + ebibleIdFromUrl, } from './ebible-license'; describe('ccLicenseFromUrl()', () => { @@ -154,6 +155,30 @@ describe('extractCcLicenseFromDetailsHtml()', () => { }); }); +describe('ebibleIdFromUrl()', () => { + const cases = [ + ['https://ebible.org/Scriptures/details.php?id=engwebp', 'engwebp'], + ['https://ebible.org/Scriptures/details.php?id=engDRA', 'engDRA'], + ['http://www.ebible.org/Scriptures/details.php?id=hin2017', 'hin2017'], + [ + 'https://EBIBLE.ORG/scriptures/DETAILS.php?foo=bar&id=arbnav', + 'arbnav', + ], + ['https://ebible.org/Scriptures/details.php', null], + ['https://ebible.org/Scriptures/details.php?id=', null], + ['https://ebible.org/find/details.php?id=engwebp', null], + ['https://example.com/Scriptures/details.php?id=engwebp', null], + ['https://berean.bible/', null], + ['not a url at all', null], + [null, null], + [undefined, null], + ] as const; + + it.each(cases)('should map %s to %s', (url, expected) => { + expect(ebibleIdFromUrl(url)).toBe(expected); + }); +}); + describe('isPublicDomainCopyrightText()', () => { const cases = [ ['This work has been dedicated to the Public Domain.', true], @@ -220,7 +245,9 @@ describe('detectEBibleLicense()', () => { }); it('should fall back to the copyright text when the fetch fails', async () => { - global.fetch = jest.fn().mockRejectedValue(new Error('network error')) as any; + global.fetch = jest + .fn() + .mockRejectedValue(new Error('network error')) as any; const license = await detectEBibleLicense({ id: 'engwmb', diff --git a/packages/helloao-cli/ebible-license.ts b/packages/helloao-cli/ebible-license.ts index ad63558..4cd99b5 100644 --- a/packages/helloao-cli/ebible-license.ts +++ b/packages/helloao-cli/ebible-license.ts @@ -109,6 +109,34 @@ export function extractCcLicenseFromDetailsHtml(html: string): License | null { return null; } +/** + * Gets the eBible translation ID from an ebible.org translation details page URL + * (e.g. `https://ebible.org/Scriptures/details.php?id=engwebp` returns `engwebp`). + * Returns null if the URL isn't an ebible.org details page URL. + */ +export function ebibleIdFromUrl(url: string | null | undefined): string | null { + if (!url) { + return null; + } + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + + const host = parsed.hostname.toLowerCase(); + if (host !== 'ebible.org' && host !== 'www.ebible.org') { + return null; + } + if (parsed.pathname.toLowerCase() !== '/scriptures/details.php') { + return null; + } + + return parsed.searchParams.get('id') || null; +} + /** * Determines whether the given ebible.org copyright notice indicates that * the translation has been dedicated to the public domain.