diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 0000000..67ee98c --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,24 @@ +name: Pylint + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pylint + - name: Analysing the code with pylint + run: | + pylint $(git ls-files '*.py') diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 256f5c7..dce822a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 diff --git a/gmail_insert.py b/gmail_insert.py index 47109dd..2f57b74 100755 --- a/gmail_insert.py +++ b/gmail_insert.py @@ -23,6 +23,7 @@ Usage: python3 gmail_insert.py < single_message.mbox + python3 gmail-insert.py --mbox < multiple_message.mbox cat archive.mbox | python3 gmail_insert.py python3 gmail_insert.py --flush # drain the spool, all users python3 gmail_insert.py --flush --user 2 # drain user 2's spool only @@ -34,6 +35,7 @@ limits the run to that one account --auth Run the interactive OAuth flow and exit --flush Re-attempt delivery of spooled messages and exit + --mbox Take potentially multiple mbox formatted messages --spool-dir DIR Where undeliverable messages are parked --on-failure spool (default) or tempfail (see --help for the retry tuning knobs) @@ -87,7 +89,7 @@ # --------------------------------------------------------------------------- SCOPES = ["https://www.googleapis.com/auth/gmail.insert"] -CREDENTIALS_FILE = "credentials.json" # OAuth2 client secret downloaded from GCP +CREDENTIALS_FILE = "credentials.json" # OAuth2 client secret downloaded from GCP SCRIPT_DIR = Path(__file__).resolve().parent @@ -198,7 +200,9 @@ def get_gmail_service(cred_id: str, *, interactive: bool = False, "Download OAuth 2.0 credentials from the Google Cloud " "Console and save them there." ) - flow = InstalledAppFlow.from_client_secrets_file(str(cred_file), SCOPES) + flow = InstalledAppFlow.from_client_secrets_file( + str(cred_file), SCOPES + ) creds = flow.run_local_server(port=0) else: raise AuthUnavailable( @@ -228,13 +232,26 @@ def split_mbox(raw: bytes) -> list[bytes]: """ Split raw mbox bytes into per-message chunks. - Eventually handle two cases: - 1. Proper mbox with one or more "From " separator lines. (Still TBD.) + Handles two cases (only invoked when --mbox given): + 1. Proper mbox with one or more "From " separator lines. 2. A bare RFC-2822 message with no "From " line (single message). """ - if not raw.strip(): - return [] - return [raw] + if raw.startswith(b"From "): + messages = [] + m_start = raw.rfind(b'\nFrom ') + m_end = len(raw) - 1 + while m_start != -1: + log.debug("[INFO] Found message at %d - %d: %s", m_start, m_end, + raw[m_start+1:m_start+20]) + messages.append(raw[m_start+1:m_end]) + m_end = m_start + m_start = raw.rfind(b'\nFrom ', 0 , m_end) + messages.append(raw[:m_end]) + log.debug("[INFO] Found message at %d - %d: %s", m_start, m_end, + raw[:20]) + return messages + else: + return [raw] def parse_headers(raw: bytes) -> EmailMessage: @@ -267,7 +284,7 @@ def describe(raw: bytes) -> dict: "date": str(msg.get("Date", "(no date)")), "message-id": str(msg.get("Message-ID", "")), } - except Exception as exc: # malformed mail must still deliver + except Exception as exc: # malformed mail must still deliver log.warning("[WARN] Could not parse headers: %s", exc) return {"from": "(unparsed)", "subject": "(unparsed)", "date": "", "message-id": ""} @@ -277,22 +294,12 @@ def describe(raw: bytes) -> dict: # Gmail insert (using the import API to get scanning and classification) # --------------------------------------------------------------------------- -def message_to_raw(raw: bytes) -> str: - """ - Base64url-encode a message for the Gmail API. - - The bytes read from stdin are forwarded verbatim (minus the mbox - envelope line) rather than round-tripped through EmailMessage.as_bytes(): - re-serialising can rewrite headers, break DKIM signatures and, on - unusual input, raise - which used to lose the message. - """ - return base64.urlsafe_b64encode(strip_envelope(raw)).decode("utf-8") - - def insert_message(service, raw: bytes) -> dict: """Insert a single message into the authenticated user's mailbox.""" body = { - "raw": message_to_raw(raw), + "raw": base64.urlsafe_b64encode( + strip_envelope(raw)).decode("utf-8" + ), "labelIds": ["INBOX", "UNREAD"], } return ( @@ -330,7 +337,7 @@ def retry_after(exc: BaseException) -> float | None: try: return max(0.0, float(value)) except (TypeError, ValueError): - return None # HTTP-date form: fall back to backoff + return None # HTTP-date form: fall back to backoff def is_retryable(exc: BaseException) -> bool: @@ -563,6 +570,7 @@ def build_arg_parser() -> argparse.ArgumentParser: python3 gmail_insert.py < message.mbox cat archive.mbox | python3 gmail_insert.py python3 gmail_insert.py --debug --user 2 < message.mbox + python3 gmail_insert.py --mbox < multiple_message.mbox python3 gmail_insert.py --json < message.mbox | jq . python3 gmail_insert.py --auth --user 2 # one-time authorisation python3 gmail_insert.py --flush # drain every user (cron) @@ -606,6 +614,11 @@ def build_arg_parser() -> argparse.ArgumentParser: "account is flushed with its own saved credentials unless " "--user is given", ) + p.add_argument( + "--mbox", + action="store_true", + help="Allow multiple messages in mbox format in the input stream" + ) p.add_argument( "--credentials", metavar="FILE", @@ -826,10 +839,13 @@ def cmd_deliver(args) -> int: log.error("[ERROR] Could not read stdin: %s", exc) return EX_TEMPFAIL - messages = split_mbox(raw_input_bytes) - if not messages: + if not raw_input_bytes.strip(): log.error("[ERROR] No messages found in input.") return 1 + if args.mbox: + messages = split_mbox(raw_input_bytes) + else: + messages = [raw_input_bytes] log.info("[INFO] Parsed %d message(s).\n", len(messages)) diff --git a/tests/driver.py b/tests/driver.py index 8fd1abf..c1fff4d 100755 --- a/tests/driver.py +++ b/tests/driver.py @@ -60,7 +60,7 @@ def fake_insert(service, raw: bytes) -> dict: # Invariant worth asserting on every delivery: the mbox envelope line # must never reach the API, and the payload must be valid base64url. - payload = base64.urlsafe_b64decode(g.message_to_raw(raw)) + payload = base64.urlsafe_b64decode(base64.urlsafe_b64encode(g.strip_envelope(raw)).decode("utf-8")) assert not payload.startswith(b"From "), "mbox envelope leaked into API payload" _record(service, raw, attempt) diff --git a/tests/test_spool.py b/tests/test_spool.py index 68d12eb..f9ac587 100644 --- a/tests/test_spool.py +++ b/tests/test_spool.py @@ -49,7 +49,7 @@ def test_envelope_line_is_synthesised_when_absent(): def test_api_payload_strips_the_envelope_line(): raw = support.message("Payload check") - payload = base64.urlsafe_b64decode(g.message_to_raw(raw)) + payload = base64.urlsafe_b64decode(base64.urlsafe_b64encode(g.strip_envelope(raw)).decode("utf-8")) support.expect(not payload.startswith(b"From "), "mbox envelope line leaked into the API payload") support.expect(payload.startswith(b"From: Sender"),