Add refresh-ebible-licenses command to update translation licenses

package.json

@@ -24,6 +24,7 @@
         "fetch:bible:commentaries": "npm run cli fetch-tyndale-open-resources -- ./sources/comment/tyndale",
         "fetch:ebible": "npm run cli source-translations -- --convert-to-usx3 ./sources/ebible",
         "import:bible:ebible": "npm run cli import-translations -- --db ./free-use-bible-api.db ./sources/ebible",
+        "refresh:ebible:licenses": "npm run cli refresh-ebible-licenses -- --db ./free-use-bible-api.db --dir ./sources/ebible",
         "import:bible:metadata": "npm run cli import-bible-metadata -- --db ./free-use-bible-api.db ./sources/metadata",
         "import:bible:commentaries": "npm run cli import-commentaries -- --db ./free-use-bible-api.db ./sources/comment",
         "zip:sync": "npm run cli upload-api-files ./free-use-bible-api.zip",

packages/helloao-cli/README.md

@@ -42,13 +42,17 @@ Commands:
   upload-api-files [options] <dest>                      Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder.
   source-translations [options] <dir> [translations...]  Finds translation sources from ebible.org and downloads it.
   list-ebible-translations [search]                      List available eBible translations. Optionally filter by search term.
+  refresh-ebible-licenses [options] [translations...]    Detects the licenses of the eBible translations that are already in the database and stores them.
+                                                         Optionally limited to the given translation IDs.
   fetch-audio [options] <dir> [translations...]          Fetches the specified audio translations and places them in the given directory.
                                                          Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
   fetch-bible-metadata <dir>                             Fetches the Theographic bible metadata and places it in the given directory.
   fetch-tyndale-open-resources <dir>                     Fetches the Tyndale Open Bible Resources and places it in the given directory.
   help [command]                                         display help for command
 ```
 
+> **eBible licenses.** `source-translations` detects the license of each eBible translation (from the Creative Commons link on its ebible.org details page, falling back to a "public domain" copyright notice) and stores it in `metadata.json`. To add licenses to a database that was imported before license detection existed, run `helloao refresh-ebible-licenses --db <path> --dir <sourced translations dir>`. It updates the `license` column of each eBible translation in place, and `--dir` also writes the licenses into the matching `metadata.json` files so that a later re-import doesn't clear them. Translations whose license can't be detected are left unchanged.
+
 > **Audio timings.** The `import-audio-timings <file>` command loads per-verse start times for a chapter's audio into the database, from a JSON array of `{ translationId, bookId, chapterNumber, reader, verses }` records. To generate that file from the audio itself, see [`tools/audio-timings`](../../tools/audio-timings/README.md), which transcribes a chapter with whisperX and aligns the result against the chapter's known verse text.
 
 The `@helloao/cli` package can also be used as a library.

packages/helloao-cli/actions.ts

@@ -14,6 +14,7 @@ import {
     bookOrder,
     dataset,
     InputTranslationMetadata,
+    License,
 } from '@helloao/tools/generation/index.js';
 import * as theographic from '@helloao/tools/generation/theographic.js';
 import fsExtra from 'fs-extra';
@@ -64,7 +65,7 @@ import {
     convertUsfmToUsx3,
 } from './conversion.js';
 import { fetchEBibleMetadata } from './ebible.js';
-import { detectEBibleLicense } from './ebible-license.js';
+import { detectEBibleLicense, ebibleIdFromUrl } from './ebible-license.js';
 import { importDatasetOutput } from './db.js';
 import { groupBy } from 'es-toolkit/compat';
 
@@ -1485,6 +1486,162 @@ export async function listEBibleTranslations(
     }
 }
 
+export interface RefreshEBibleLicensesOptions {
+    /**
+     * The database to update.
+     */
+    db?: string | null;
+
+    /**
+     * The directory that contains the sourced translations (e.g. the output directory of source-translations).
+     * If provided, the metadata.json files in this directory will also be updated with the detected licenses
+     * so that the licenses are preserved when the translations are re-imported.
+     */
+    dir?: string;
+}
+
+/**
+ * Detects the licenses of the eBible translations that are already in the database and stores them.
+ * This avoids having to re-download and re-import translations just to pick up license information.
+ * @param translations The IDs of the translations that should be refreshed. If empty, all eBible translations are refreshed.
+ * @param options The options.
+ */
+export async function refreshEBibleLicenses(
+    translations: string[],
+    options: RefreshEBibleLicensesOptions
+): Promise<void> {
+    const logger = log.getLogger();
+    const db = await database.getDb(options.db);
+    try {
+        const rows = db
+            .prepare('SELECT id, licenseUrl, website FROM Translation;')
+            .all() as { id: string; licenseUrl: string; website: string }[];
+
+        const requested = new Set(translations);
+        const targets: { id: string; ebibleId: string }[] = [];
+        for (let row of rows) {
+            if (requested.size > 0 && !requested.has(row.id)) {
+                continue;
+            }
+            const ebibleId =
+                ebibleIdFromUrl(row.licenseUrl) ?? ebibleIdFromUrl(row.website);
+            if (ebibleId) {
+                targets.push({ id: row.id, ebibleId });
+            } else if (requested.has(row.id)) {
+                logger.warn(
+                    `${row.id} is not an eBible translation. Skipping.`
+                );
+            }
+        }
+
+        for (let id of requested) {
+            if (!rows.some((r) => r.id === id)) {
+                logger.warn(`${id} was not found in the database. Skipping.`);
+            }
+        }
+
+        if (targets.length === 0) {
+            logger.log('No eBible translations found to refresh.');
+            return;
+        }
+
+        // The copyright text is used as a fallback when the details page has no Creative Commons license.
+        const copyrights = new Map<string, string>();
+        const storedSources = db
+            .prepare('SELECT id, copyright FROM EBibleSource;')
+            .all() as { id: string; copyright: string }[];
+        for (let source of storedSources) {
+            copyrights.set(source.id.toLowerCase(), source.copyright);
+        }
+        try {
+            for (let source of await fetchEBibleMetadata()) {
+                copyrights.set(source.id.toLowerCase(), source.copyright);
+            }
+        } catch (err) {
+            logger.warn(
+                'Failed to fetch the eBible translation list. Using the copyright text stored in the database.',
+                err
+            );
+        }
+
+        logger.log(`Refreshing licenses for ${targets.length} translations...`);
+
+        const updateLicense = db.prepare(
+            'UPDATE Translation SET license = @license WHERE id = @id;'
+        );
+        const licenses = new Map<string, License>();
+        for (let [index, target] of targets.entries()) {
+            const progress = `[${index + 1}/${targets.length}]`;
+            const license = await detectEBibleLicense({
+                id: target.ebibleId,
+                copyright: copyrights.get(target.ebibleId.toLowerCase()) ?? '',
+            });
+
+            if (!license) {
+                logger.log(
+                    `${progress} ${target.id}: No license detected. Leaving unchanged.`
+                );
+                continue;
+            }
+
+            updateLicense.run({
+                id: target.id,
+                license: JSON.stringify(license),
+            });
+            licenses.set(target.id, license);
+            logger.log(`${progress} ${target.id}: ${JSON.stringify(license)}`);
+        }
+
+        logger.log(
+            `Updated licenses for ${licenses.size} of ${targets.length} translations.`
+        );
+
+        if (options.dir) {
+            const updatedFiles = await updateMetadataLicenses(
+                options.dir,
+                licenses
+            );
+            logger.log(
+                `Updated ${updatedFiles} metadata.json files in ${options.dir}.`
+            );
+        }
+    } finally {
+        db.close();
+    }
+}
+
+/**
+ * Sets the license in the metadata.json file of each translation in the given directory.
+ * @param dir The directory that contains the translation directories.
+ * @param licenses The map of translation IDs to their licenses.
+ * @returns The number of metadata.json files that were updated.
+ */
+async function updateMetadataLicenses(
+    dir: string,
+    licenses: Map<string, License>
+): Promise<number> {
+    let updated = 0;
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (let entry of entries) {
+        if (!entry.isDirectory()) {
+            continue;
+        }
+        const metadataPath = path.resolve(dir, entry.name, 'metadata.json');
+        if (!(await exists(metadataPath))) {
+            continue;
+        }
+        const metadata = JSON.parse(await readFile(metadataPath, 'utf-8'));
+        const license = licenses.get(metadata.id);
+        if (!license) {
+            continue;
+        }
+        metadata.license = license;
+        await writeFile(metadataPath, JSON.stringify(metadata, null, 2));
+        updated++;
+    }
+    return updated;
+}
+
 /**
  * Generates the translation files directly from the translation stored in the given input directory.
  * @param input The input directory that the translation is stored in.

packages/helloao-cli/cli.ts

@@ -25,6 +25,7 @@ import {
     importTranslations,
     initDb,
     listEBibleTranslations,
+    refreshEBibleLicenses,
     sourceTranslations,
     searchTypesenseVerses,
     uploadTestTranslation,
@@ -902,6 +903,22 @@ async function start() {
             await listEBibleTranslations(search);
         });
 
+    program
+        .command('refresh-ebible-licenses [translations...]')
+        .description(
+            'Detects the licenses of the eBible translations that are already in the database and stores them.\nOptionally limited to the given translation IDs.'
+        )
+        .option(
+            '--dir <dir>',
+            'The directory that the translations were sourced into. If provided, their metadata.json files are also updated so that the licenses are kept when they are re-imported.'
+        )
+        .action(async (translations: string[], options: any) => {
+            await refreshEBibleLicenses(translations, {
+                ...program.opts(),
+                ...options,
+            });
+        });
+
     program
         .command('fetch-audio <dir> [translations...]')
         .description(

packages/helloao-cli/ebible-license.spec.ts

@@ -3,6 +3,7 @@ import {
     extractCcLicenseFromDetailsHtml,
     isPublicDomainCopyrightText,
     detectEBibleLicense,
+    ebibleIdFromUrl,
 } from './ebible-license';
 
 describe('ccLicenseFromUrl()', () => {
@@ -154,6 +155,30 @@ describe('extractCcLicenseFromDetailsHtml()', () => {
     });
 });
 
+describe('ebibleIdFromUrl()', () => {
+    const cases = [
+        ['https://ebible.org/Scriptures/details.php?id=engwebp', 'engwebp'],
+        ['https://ebible.org/Scriptures/details.php?id=engDRA', 'engDRA'],
+        ['http://www.ebible.org/Scriptures/details.php?id=hin2017', 'hin2017'],
+        [
+            'https://EBIBLE.ORG/scriptures/DETAILS.php?foo=bar&id=arbnav',
+            'arbnav',
+        ],
+        ['https://ebible.org/Scriptures/details.php', null],
+        ['https://ebible.org/Scriptures/details.php?id=', null],
+        ['https://ebible.org/find/details.php?id=engwebp', null],
+        ['https://example.com/Scriptures/details.php?id=engwebp', null],
+        ['https://berean.bible/', null],
+        ['not a url at all', null],
+        [null, null],
+        [undefined, null],
+    ] as const;
+
+    it.each(cases)('should map %s to %s', (url, expected) => {
+        expect(ebibleIdFromUrl(url)).toBe(expected);
+    });
+});
+
 describe('isPublicDomainCopyrightText()', () => {
     const cases = [
         ['This work has been dedicated to the Public Domain.', true],
@@ -220,7 +245,9 @@ describe('detectEBibleLicense()', () => {
     });
 
     it('should fall back to the copyright text when the fetch fails', async () => {
-        global.fetch = jest.fn().mockRejectedValue(new Error('network error')) as any;
+        global.fetch = jest
+            .fn()
+            .mockRejectedValue(new Error('network error')) as any;
 
         const license = await detectEBibleLicense({
             id: 'engwmb',

packages/helloao-cli/ebible-license.ts

@@ -109,6 +109,34 @@ export function extractCcLicenseFromDetailsHtml(html: string): License | null {
     return null;
 }
 
+/**
+ * Gets the eBible translation ID from an ebible.org translation details page URL
+ * (e.g. `https://ebible.org/Scriptures/details.php?id=engwebp` returns `engwebp`).
+ * Returns null if the URL isn't an ebible.org details page URL.
+ */
+export function ebibleIdFromUrl(url: string | null | undefined): string | null {
+    if (!url) {
+        return null;
+    }
+
+    let parsed: URL;
+    try {
+        parsed = new URL(url);
+    } catch {
+        return null;
+    }
+
+    const host = parsed.hostname.toLowerCase();
+    if (host !== 'ebible.org' && host !== 'www.ebible.org') {
+        return null;
+    }
+    if (parsed.pathname.toLowerCase() !== '/scriptures/details.php') {
+        return null;
+    }
+
+    return parsed.searchParams.get('id') || null;
+}
+
 /**
  * Determines whether the given ebible.org copyright notice indicates that
  * the translation has been dedicated to the public domain.