From 42d2fcb9746976de128a8172a8b7658e1ed93dce Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 12:51:33 -0400 Subject: [PATCH 01/14] Add Pylint workflow for Python code analysis --- .github/workflows/pylint.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/pylint.yml diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..c73e032 --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,23 @@ +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pylint + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') From 3ea843c0460643b53793d5e3da1aa42b6aa7a587 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 12:55:03 -0400 Subject: [PATCH 02/14] Create tests.yml --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1 @@ + From babc36aadb5248b2a3884cf1a61ae0af4e4c8e03 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 13:03:25 -0400 Subject: [PATCH 03/14] Create tests workflow for regression testing Add GitHub Actions workflow for running tests on multiple Python versions. --- .github/workflows/tests.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b13789..3873d43 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1 +1,28 @@ +name: tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + regression: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # No install step on purpose: the suite stubs the Google client + # libraries, so a regression run needs nothing but the interpreter. + - name: Run the regression suite + run: python3 gmail_insert.py -h From e0954195cdb5c29615b050500a4896fa2c6a33cb Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 13:10:14 -0400 Subject: [PATCH 04/14] Add installation of dependencies in pylint workflow --- .github/workflows/pylint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index c73e032..1c94165 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -17,6 +17,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + pip install -r requirements.txt pip install pylint - name: Analysing the code with pylint run: | From ff58bc5e36c35ea1822a92b05fda2a26e3fa75d4 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 13:20:23 -0400 Subject: [PATCH 05/14] Update tests.yml --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3873d43..c38bf7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,9 +16,9 @@ jobs: python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} From 797d25423d80931ccfc41965f68181740ac8d64c Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 13:21:08 -0400 Subject: [PATCH 06/14] Update Python versions in pylint workflow --- .github/workflows/pylint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 1c94165..0624998 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -7,11 +7,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.111", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 803c9165ff2429a3f2bbd535d37455ffe532a330 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 13:24:25 -0400 Subject: [PATCH 07/14] Fix Python version formatting in pylint workflow --- .github/workflows/pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 0624998..a1f09a5 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.111", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 1c4587224fecc028fb2d1c0289db94887246f467 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 17:00:10 -0400 Subject: [PATCH 08/14] Fix typo in test script execution command --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c38bf7b..0bec6de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,4 +25,4 @@ jobs: # No install step on purpose: the suite stubs the Google client # libraries, so a regression run needs nothing but the interpreter. - name: Run the regression suite - run: python3 gmail_insert.py -h + run: ppython3 tests/run_tests.py From 403b26d50b0ce8d88e0a4ed06bf73fc38df23040 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 17:00:26 -0400 Subject: [PATCH 09/14] Fix typo in command to run regression tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0bec6de..9e420a3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,4 +25,4 @@ jobs: # No install step on purpose: the suite stubs the Google client # libraries, so a regression run needs nothing but the interpreter. - name: Run the regression suite - run: ppython3 tests/run_tests.py + run: python3 tests/run_tests.py From 670a06dbeb4fc3cfb973208a999f364f1a53c373 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 22:19:59 -0400 Subject: [PATCH 10/14] Update pylint.yml --- .github/workflows/pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index a1f09a5..67ee98c 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -9,7 +9,7 @@ jobs: matrix: python-version: ["3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v7 with: From 29733c7846cc871fd0c9e35366c90fe4323686c5 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Sun, 2 Aug 2026 22:49:26 -0400 Subject: [PATCH 11/14] Add Python 3.14 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 256f5c7..dce822a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 From 5d35fce0b60bc83d0492959a89d954d9d3b42c15 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Mon, 3 Aug 2026 00:19:54 -0400 Subject: [PATCH 12/14] Simplify the path for base64 encoding --- gmail_insert.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/gmail_insert.py b/gmail_insert.py index 47109dd..8375386 100755 --- a/gmail_insert.py +++ b/gmail_insert.py @@ -87,7 +87,7 @@ # --------------------------------------------------------------------------- SCOPES = ["https://www.googleapis.com/auth/gmail.insert"] -CREDENTIALS_FILE = "credentials.json" # OAuth2 client secret downloaded from GCP +CREDENTIALS_FILE = "credentials.json" # OAuth2 client secret downloaded from GCP SCRIPT_DIR = Path(__file__).resolve().parent @@ -198,7 +198,9 @@ def get_gmail_service(cred_id: str, *, interactive: bool = False, "Download OAuth 2.0 credentials from the Google Cloud " "Console and save them there." ) - flow = InstalledAppFlow.from_client_secrets_file(str(cred_file), SCOPES) + flow = InstalledAppFlow.from_client_secrets_file( + str(cred_file), SCOPES + ) creds = flow.run_local_server(port=0) else: raise AuthUnavailable( @@ -267,7 +269,7 @@ def describe(raw: bytes) -> dict: "date": str(msg.get("Date", "(no date)")), "message-id": str(msg.get("Message-ID", "")), } - except Exception as exc: # malformed mail must still deliver + except Exception as exc: # malformed mail must still deliver log.warning("[WARN] Could not parse headers: %s", exc) return {"from": "(unparsed)", "subject": "(unparsed)", "date": "", "message-id": ""} @@ -277,22 +279,12 @@ def describe(raw: bytes) -> dict: # Gmail insert (using the import API to get scanning and classification) # --------------------------------------------------------------------------- -def message_to_raw(raw: bytes) -> str: - """ - Base64url-encode a message for the Gmail API. - - The bytes read from stdin are forwarded verbatim (minus the mbox - envelope line) rather than round-tripped through EmailMessage.as_bytes(): - re-serialising can rewrite headers, break DKIM signatures and, on - unusual input, raise - which used to lose the message. - """ - return base64.urlsafe_b64encode(strip_envelope(raw)).decode("utf-8") - - def insert_message(service, raw: bytes) -> dict: """Insert a single message into the authenticated user's mailbox.""" body = { - "raw": message_to_raw(raw), + "raw": base64.urlsafe_b64encode( + strip_envelope(raw)).decode("utf-8" + ), "labelIds": ["INBOX", "UNREAD"], } return ( @@ -330,7 +322,7 @@ def retry_after(exc: BaseException) -> float | None: try: return max(0.0, float(value)) except (TypeError, ValueError): - return None # HTTP-date form: fall back to backoff + return None # HTTP-date form: fall back to backoff def is_retryable(exc: BaseException) -> bool: From 858591203717a014ac09ba340fd48b4cdbe8324f Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Mon, 3 Aug 2026 02:21:46 -0400 Subject: [PATCH 13/14] Implement handling full mboxes --- gmail_insert.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/gmail_insert.py b/gmail_insert.py index 8375386..2f57b74 100755 --- a/gmail_insert.py +++ b/gmail_insert.py @@ -23,6 +23,7 @@ Usage: python3 gmail_insert.py < single_message.mbox + python3 gmail-insert.py --mbox < multiple_message.mbox cat archive.mbox | python3 gmail_insert.py python3 gmail_insert.py --flush # drain the spool, all users python3 gmail_insert.py --flush --user 2 # drain user 2's spool only @@ -34,6 +35,7 @@ limits the run to that one account --auth Run the interactive OAuth flow and exit --flush Re-attempt delivery of spooled messages and exit + --mbox Take potentially multiple mbox formatted messages --spool-dir DIR Where undeliverable messages are parked --on-failure spool (default) or tempfail (see --help for the retry tuning knobs) @@ -230,13 +232,26 @@ def split_mbox(raw: bytes) -> list[bytes]: """ Split raw mbox bytes into per-message chunks. - Eventually handle two cases: - 1. Proper mbox with one or more "From " separator lines. (Still TBD.) + Handles two cases (only invoked when --mbox given): + 1. Proper mbox with one or more "From " separator lines. 2. A bare RFC-2822 message with no "From " line (single message). """ - if not raw.strip(): - return [] - return [raw] + if raw.startswith(b"From "): + messages = [] + m_start = raw.rfind(b'\nFrom ') + m_end = len(raw) - 1 + while m_start != -1: + log.debug("[INFO] Found message at %d - %d: %s", m_start, m_end, + raw[m_start+1:m_start+20]) + messages.append(raw[m_start+1:m_end]) + m_end = m_start + m_start = raw.rfind(b'\nFrom ', 0 , m_end) + messages.append(raw[:m_end]) + log.debug("[INFO] Found message at %d - %d: %s", m_start, m_end, + raw[:20]) + return messages + else: + return [raw] def parse_headers(raw: bytes) -> EmailMessage: @@ -555,6 +570,7 @@ def build_arg_parser() -> argparse.ArgumentParser: python3 gmail_insert.py < message.mbox cat archive.mbox | python3 gmail_insert.py python3 gmail_insert.py --debug --user 2 < message.mbox + python3 gmail_insert.py --mbox < multiple_message.mbox python3 gmail_insert.py --json < message.mbox | jq . python3 gmail_insert.py --auth --user 2 # one-time authorisation python3 gmail_insert.py --flush # drain every user (cron) @@ -598,6 +614,11 @@ def build_arg_parser() -> argparse.ArgumentParser: "account is flushed with its own saved credentials unless " "--user is given", ) + p.add_argument( + "--mbox", + action="store_true", + help="Allow multiple messages in mbox format in the input stream" + ) p.add_argument( "--credentials", metavar="FILE", @@ -818,10 +839,13 @@ def cmd_deliver(args) -> int: log.error("[ERROR] Could not read stdin: %s", exc) return EX_TEMPFAIL - messages = split_mbox(raw_input_bytes) - if not messages: + if not raw_input_bytes.strip(): log.error("[ERROR] No messages found in input.") return 1 + if args.mbox: + messages = split_mbox(raw_input_bytes) + else: + messages = [raw_input_bytes] log.info("[INFO] Parsed %d message(s).\n", len(messages)) From df78bcce82d4c02b64fec0106158edbe0859bed0 Mon Sep 17 00:00:00 2001 From: Howard Holm Date: Mon, 3 Aug 2026 02:43:09 -0400 Subject: [PATCH 14/14] Fix the tests to account for the simplification of the code --- tests/driver.py | 2 +- tests/test_spool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/driver.py b/tests/driver.py index 8fd1abf..c1fff4d 100755 --- a/tests/driver.py +++ b/tests/driver.py @@ -60,7 +60,7 @@ def fake_insert(service, raw: bytes) -> dict: # Invariant worth asserting on every delivery: the mbox envelope line # must never reach the API, and the payload must be valid base64url. - payload = base64.urlsafe_b64decode(g.message_to_raw(raw)) + payload = base64.urlsafe_b64decode(base64.urlsafe_b64encode(g.strip_envelope(raw)).decode("utf-8")) assert not payload.startswith(b"From "), "mbox envelope leaked into API payload" _record(service, raw, attempt) diff --git a/tests/test_spool.py b/tests/test_spool.py index 68d12eb..f9ac587 100644 --- a/tests/test_spool.py +++ b/tests/test_spool.py @@ -49,7 +49,7 @@ def test_envelope_line_is_synthesised_when_absent(): def test_api_payload_strips_the_envelope_line(): raw = support.message("Payload check") - payload = base64.urlsafe_b64decode(g.message_to_raw(raw)) + payload = base64.urlsafe_b64decode(base64.urlsafe_b64encode(g.strip_envelope(raw)).decode("utf-8")) support.expect(not payload.startswith(b"From "), "mbox envelope line leaked into the API payload") support.expect(payload.startswith(b"From: Sender"),