TIMX 655 - handle deletes for researchdatabases

README.md

@@ -59,8 +59,27 @@ WORKSPACE=### Set to `dev` for local development, this will be set to `stage` an
 WARNING_ONLY_LOGGERS=### Comma-seperated list of logger names to set as WARNING only, e.g. 'botocore,charset_normalizer,smart_open'
 LIBGUIDES_API_TOKEN=### Libguides API token [required for libguides source]
 LIBGUIDES_CLIENT_ID=### Libguides account id [required for libguides source] 
+LAST_AZ_IDENTIFIERS_PATH=### S3 or local filepath of the last known public AZ item identifiers [enables synthetic delete records for researchdatabases source]
 ```
 
+#### `LAST_AZ_IDENTIFIERS_PATH`
+
+The Springshare OAI-PMH endpoint does not emit deletes for AZ (research database) items
+that have been unpublished or hidden.  When `LAST_AZ_IDENTIFIERS_PATH` is set, the
+`researchdatabases` transformation queries the LibGuides API for the current set of
+public AZ items, compares it to the identifiers stored at this path, and yields synthetic
+delete records for any identifiers that have dropped out.  The file is then rewritten with
+the current identifiers for the next run.
+
+Notes:
+
+- If the env var is not set, no deletes are determined and the LibGuides API is not
+queried.  This makes the behavior opt-in and backwards compatible with runs that predate
+it.
+- If the env var is set but the file does not yet exist, that run cannot determine any
+deletes, but it will create the file so that subsequent runs can.
+- This also requires `LIBGUIDES_API_TOKEN` and `LIBGUIDES_CLIENT_ID` to be set.
+
 ## CLI commands
 
 ### `transform`

tests/fixtures/researchdatabases/researchdatabases-2026-08-11-full-extracted-records-to-index.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+    <responseDate>2026-08-11T09:30:27Z</responseDate>
+    <request verb="ListRecords" metadataPrefix="oai_dc" set="az">
+        https://libguides.mit.edu/oai.php
+    </request>
+    <ListRecords>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257807</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title>
+                        <![CDATA[Linguistics and Language Behavior Abstracts (LLBA)]]></dc:title>
+                    <dc:subject><![CDATA[Humanities]]></dc:subject>
+                    <dc:type><![CDATA[Articles]]></dc:type>
+                    <dc:type><![CDATA[Dissertations & theses]]></dc:type>
+                    <dc:type><![CDATA[eBooks]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p>The most comprehensive index to articles in Linguistics and Language Development and use.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/llba</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257808</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title><![CDATA[ADS (NASA Astrophysics Data System)]]></dc:title>
+                    <dc:subject><![CDATA[Engineering]]></dc:subject>
+                    <dc:subject><![CDATA[Science]]></dc:subject>
+                    <dc:type><![CDATA[Articles]]></dc:type>
+                    <dc:type><![CDATA[Data & statistics]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p>Indexes major astronomy and astrophysics journals; includes abstracts for most entries and full text scans of tens of thousands of articles.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/ads</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257809</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title><![CDATA[Ethnic NewsWatch]]></dc:title>
+                    <dc:subject><![CDATA[Social sciences]]></dc:subject>
+                    <dc:type><![CDATA[Newspapers & magazine articles]]></dc:type>
+                    <dc:type><![CDATA[Primary sources]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p><strong>Coverage:</strong> 1990 - present<br>A full-text collection of newspapers, journals and magazines published by ethnic and minority presses.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/ethnic</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+    </ListRecords>
+</OAI-PMH>

tests/sources/json/test_libguides.py

@@ -401,7 +401,7 @@ def test_libguides_api_client_fetch_guides_expands_sub_pages_into_rows():
     mock_response.json.return_value = mock_api_response
 
     with patch(
-        "transmogrifier.sources.json.libguides.requests.get",
+        "transmogrifier.helpers.requests.get",
         return_value=mock_response,
     ):
         df = client.fetch_guides("fake-token")

tests/sources/xml/test_researchdatabases.py

@@ -0,0 +1,161 @@
+# ruff: noqa: PLR2004, SLF001
+
+from unittest.mock import patch
+
+import pytest
+from bs4 import Tag
+
+from transmogrifier.helpers import LibGuidesAPIClient
+from transmogrifier.sources.xml.researchdatabases import ResearchDatabases
+
+
+@pytest.fixture
+def last_az_identifiers(tmp_path) -> str:
+    path = tmp_path / "last_az_identifiers.txt"
+    with open(path, "w") as f:
+        f.write("1234")
+    return str(path)
+
+
+@pytest.fixture(autouse=True)
+def _test_env_libguides(last_az_identifiers):
+    with (
+        patch("transmogrifier.config.LIBGUIDES_CLIENT_ID", "123"),
+        patch("transmogrifier.config.LIBGUIDES_API_TOKEN", "aaabbbdddccc"),
+        patch(
+            "transmogrifier.sources.xml.researchdatabases.LAST_AZ_IDENTIFIERS_PATH",
+            last_az_identifiers,
+        ),
+    ):
+        yield
+
+
+@pytest.fixture
+def mocked_current_az_identifiers():
+    """Mock current AZ identifiers from LibGuides API, none matching previous ones."""
+    with patch.object(
+        LibGuidesAPIClient,
+        "get_current_az_identifiers",
+        return_value=["7777", "8888", "9999"],
+    ):
+        yield
+
+
+@pytest.fixture
+def researchdatabases_transformer():
+
+    return ResearchDatabases.load(
+        "researchdatabases",
+        (
+            "tests/fixtures/researchdatabases/researchdatabases-"
+            "2026-08-11-full-extracted-records-to-index.xml"
+        ),
+    )
+
+
+def test_researchdatabases_yields_oai_records_pass(researchdatabases_transformer):
+
+    oai_records = list(
+        researchdatabases_transformer._yield_oai_xml_records_for_indexing(
+            researchdatabases_transformer.source_file
+        )
+    )
+
+    assert len(oai_records) == 3
+    assert isinstance(oai_records[0], Tag)
+
+
+def test_researchdatabases_yields_deleted_records_success(
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    deleted_records = list(
+        researchdatabases_transformer._yield_api_records_for_deleting()
+    )
+    assert len(deleted_records) == 1
+    assert isinstance(deleted_records[0], Tag)
+
+
+def test_researchdatabases_synthetic_deleted_record(
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    """Assert that Transformer injects synthetic OAI delete records.
+
+    Example record:
+
+    <?xml version="1.0" encoding="utf-8"?>
+    <record>
+        <header status="deleted">
+            <identifier>oai:libguides.com:az/1234</identifier>
+            <datestamp>2026-08-11T14:13:38Z</datestamp>
+            <setSpec>az</setSpec>
+        </header>
+        <note>This is a synthetic delete record created by Transmogrifier.</note>
+        <metadata/>
+    </record>
+    """
+    deleted_record = next(researchdatabases_transformer._yield_api_records_for_deleting())
+
+    header = deleted_record.find("record").find("header")
+
+    assert header.get("status") == "deleted"
+    assert header.find("identifier").string == "oai:libguides.com:az/1234"
+    assert (
+        deleted_record.find("note").string
+        == "This is a synthetic delete record created by Transmogrifier."
+    )
+
+
+def test_researchdatabases_last_az_identifiers_file_updated(
+    researchdatabases_transformer, mocked_current_az_identifiers, last_az_identifiers
+):
+    """Assert the last AZ identifiers list is updated."""
+    _ = list(researchdatabases_transformer._yield_api_records_for_deleting())
+    with open(last_az_identifiers) as f:
+        assert f.read() == """7777\n8888\n9999\n"""
+
+
+def test_researchdatabases_env_var_not_set_skips_deletes(
+    caplog,
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    """Assert that deletes are opt-in via the 'LAST_AZ_IDENTIFIERS_PATH' env var."""
+    caplog.set_level("WARNING")
+    with patch(
+        "transmogrifier.sources.xml.researchdatabases.LAST_AZ_IDENTIFIERS_PATH",
+        None,
+    ):
+        assert list(researchdatabases_transformer._yield_api_records_for_deleting()) == []
+
+    assert "Env var 'LAST_AZ_IDENTIFIERS_PATH' is not set" in caplog.text
+
+
+def test_researchdatabases_identifiers_text_file_missing_creates_file(
+    caplog,
+    tmp_path,
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    """Assert that a missing AZ identifiers list yields no deletes, but creates file."""
+    caplog.set_level("WARNING")
+    cold_start_path = tmp_path / "does-not-exist.txt"
+    with patch(
+        "transmogrifier.sources.xml.researchdatabases.LAST_AZ_IDENTIFIERS_PATH",
+        str(cold_start_path),
+    ):
+        assert list(researchdatabases_transformer._yield_api_records_for_deleting()) == []
+
+    assert "but file does not exist, this run will create it" in caplog.text
+    with open(cold_start_path) as f:
+        assert f.read() == "7777\n8888\n9999\n"
+
+
+def test_research_databases_custom_parse_source_file_yields_oai_and_synthetic_records(
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    records = list(researchdatabases_transformer.source_records)
+
+    assert len(records) == 4  # 3 OAI + 1 synthetic delete

tests/test_cli.py

@@ -215,9 +215,9 @@ def test_transform_no_memory_fault_for_threaded_bs4_parsing(monkeypatch, tmp_pat
             "run",
             "transform",
             "-s",
-            "researchdatabases",
+            "libguides",
             "-i",
-            "tests/fixtures/dataset/libguides-2025-01-09-full-extracted-records-to-index.xml",
+            "tests/fixtures/libguides/libguides-2026-02-20-full-extracted-records-to-index.jsonl",
             "-o",
             f"{tmp_path}/dataset",
         ],

transmogrifier/config.py

@@ -127,7 +127,7 @@
     "researchdatabases": {
         "name": "Research Databases",
         "base-url": "https://libguides.mit.edu/",
-        "transform-class": "transmogrifier.sources.xml.springshare.SpringshareOaiDc",
+        "transform-class": "transmogrifier.sources.xml.researchdatabases.ResearchDatabases",  # noqa: E501
     },
     "whoas": {
         "name": "Woods Hole Open Access Server",
@@ -149,6 +149,7 @@
 )
 LIBGUIDES_API_TOKEN = os.getenv("LIBGUIDES_API_TOKEN")
 LIBGUIDES_CLIENT_ID = os.getenv("LIBGUIDES_CLIENT_ID")
+LAST_AZ_IDENTIFIERS_PATH = os.getenv("LAST_AZ_IDENTIFIERS_PATH")
 
 
 def configure_logger(