diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37885f4..9e35b84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,27 @@ jobs: exit 1 fi + dist: + name: Build, install and run the distribution + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: ${{ env.UV_VERSION }} + python-version: "3.12" + enable-cache: false + + - name: Build, twine check, install the wheel in a clean environment, run it + # Uploads nothing. publish-pypi.yml is the only workflow that can. + run: make dist-check + scan: name: Secret, static and dependency scanning runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f29f3f..0676fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **An installed `ca-tariff-parse` failed on import.** `export.py` read the + published schema from the repository root, two directories above the module. + In a checkout that is the schema; in a wheel it is `lib/python3.x/`, so every + command, `--help` and `--version` included, died with `ExportError` before + printing anything. The wheel now carries a copy under + `ca_tariff_parse/_schemas/` and a checkout still reads the one published file + at `schemas/`. The `v0.3.0` tag predates the fix, so `uv tool install + "git+...@v0.3.0"` and a wheel built from that tag both have the defect; the + next release is the first that does not. Nothing had been uploaded to an + index, so no published release carries it. +- The wheel's `[project.urls]` gains `Changelog`, `Documentation` and + `Security`, beside the existing `Homepage`, `Source` and `Issues`. +- `make dist-check` (and a `dist` job in CI) builds the sdist and wheel, runs + `twine check --strict`, installs the wheel into an empty environment outside + the checkout, runs the CLI, and asserts the Project-URL metadata. Nothing is + uploaded. + - **`E-TOU-C` read zero time-of-use windows, and it is the schedule named after them (#75).** The sheet states four -- Peak and Off-Peak under each of two seasons -- as a list under a season heading rather than as the three-column diff --git a/Makefile b/Makefile index 79edb6e..834d106 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SOURCES_DIR ?= sources .DEFAULT_GOAL := help .PHONY: help install lint fmt typecheck test verify coverage coverage-real fetch verify-source \ - golden watch watch-log watch-baseline clean + golden watch watch-log watch-baseline dist-check clean help: ## Show this help @grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) \ @@ -29,6 +29,9 @@ test: ## Run the test suite with the coverage floor verify: install lint typecheck test ## Everything CI runs +dist-check: ## Build sdist and wheel, install the wheel in a clean venv, run it (uploads nothing) + scripts/dist-check.sh + coverage: ## Report parser coverage of the synthetic fixtures $(UV) run ca-tariff-parse coverage tests/fixtures/SYNTHETIC-example-schedule-complete.txt $(UV) run ca-tariff-parse coverage \ diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..01080ff --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,133 @@ +# Releasing to PyPI + +Nothing in this repository publishes to PyPI on its own. This is the procedure +a maintainer follows, written so each step can be followed without reading the +workflows first, with what it should look like when it worked. + +Last verified: 2026-10-17. `https://pypi.org/pypi/ca-tariff-parse/json` and the +TestPyPI equivalent both returned 404, so the name is free but not reserved. +Recheck before step 2: a 404 today says nothing about next week. + +## What was checked, and what it found + +Run `make dist-check` to repeat it. It builds the sdist and wheel, runs +`twine check --strict`, builds a wheel again from the sdist, installs the wheel +into an empty virtual environment outside the checkout, runs the command line +tool and asserts the `Project-URL` metadata the wheel carries. + +| Item | State | +| --- | --- | +| Name, license, readme, classifiers, `requires-python`, console script | Present in `pyproject.toml` and in the built `METADATA`. | +| `Project-URL` metadata | `Homepage`, `Source`, `Issues`, `Changelog`, `Documentation`, `Security`. The last three were added; a published release's metadata cannot be edited afterwards. | +| Clean-environment install | **Failed before this change.** The `v0.3.0` wheel died on import with `ExportError`, because the published schema was read from the repository root and the wheel did not carry it. Fixed in `src/ca_tariff_parse/export.py` and `pyproject.toml`. | +| Release workflow | Already present: `release.yml` (signed tag to draft GitHub release) and `publish-pypi.yml` (dispatch-only, OIDC trusted publishing, no stored token). Neither was changed. | + +### The v0.3.0 tag must not be published + +`publish-pypi.yml` builds at the tagged commit. `v0.3.0` predates the schema fix, +so uploading it would put a wheel on PyPI that cannot start, and an uploaded +release is immutable: PyPI will not accept the same filename again, even after +deletion. The same defect makes the README's +`uv tool install "git+https://github.com/ChelseaKR/ca-tariff-parse@v0.3.0"` +fail today. Cut a new version first. `[Unreleased]` in `CHANGELOG.md` carries +new features as well as the fix, so a minor bump (`0.4.0`) matches its contents. + +## Procedure + +### 1. Merge the readiness pull request + +It carries the fix above and `make dist-check`. Confirm the `dist` job in CI is +green on `main`. + +### 2. Register the project on PyPI (once) + +A maintainer signs in at pypi.org and opens *Your account*, then *Publishing*, then +*Add a new pending publisher*, GitHub tab. Enter exactly: + +| Field | Value | +| --- | --- | +| PyPI Project Name | `ca-tariff-parse` | +| Owner | `ChelseaKR` | +| Repository name | `ca-tariff-parse` | +| Workflow name | `publish-pypi.yml` | +| Environment name | `pypi` | + +`Workflow name` is the filename, not the workflow's `name:` field. A typo there +fails like a permissions problem. A pending publisher reserves nothing until the +first upload succeeds, so do step 5 soon after. + +The account needs two-factor authentication, which PyPI requires for publishing. + +### 3. Create the `pypi` environment on GitHub (once) + +Repository *Settings*, then *Environments*, then *New environment*, named `pypi`. +Add a maintainer under *Required reviewers* so the upload waits for a +confirmation. Do not add secrets. There is no token and none is wanted. + +### 4. Prepare and tag the release + +Follow the shape of release PR #36 (commit `95bf997`). In one pull request: + +1. `pyproject.toml`: `version`. +2. `src/ca_tariff_parse/parser.py`: `PARSER_VERSION`, same string. +3. `CITATION.cff`: `version` and `date-released`. +4. `CHANGELOG.md`: turn `[Unreleased]` into `[X.Y.Z] - YYYY-MM-DD` and open a + new empty `[Unreleased]`. +5. `README.md`: the `@v0.3.0` pins in *Quick start*. +6. Regenerate what stamps the parser version: `data/parsed/*.json` and + `tests/golden/*.json` change in that one field and no other. Review that the + diff says only that. `uv lock` brings `uv.lock` along. +7. `make verify` and `make dist-check` both exit 0. + +After it merges, tag the merge commit with a signing key whose public half is in +`.github/allowed_signers`: + +```bash +git checkout main && git pull +git tag -s vX.Y.Z -m "vX.Y.Z" +git verify-tag vX.Y.Z # needs gpg.format ssh and gpg.ssh.allowedSignersFile +git push origin vX.Y.Z +``` + +### 5. Release, then publish + +1. *Actions*, then `release`, *Run workflow* from `main`, input `vX.Y.Z`. It + re-verifies the signature, re-runs `make verify` at the tagged commit and + creates a **draft** GitHub release. +2. Read the draft on the *Releases* page and press *Publish release*. + `publish-pypi.yml` refuses a tag whose release is still a draft. +3. *Actions*, then `publish-pypi`, *Run workflow* from `main`, input `vX.Y.Z`. + Approve the `pypi` environment prompt when it appears. + +Expected: the `publish` job goes green and +`https://pypi.org/project/ca-tariff-parse/X.Y.Z/` exists, with attestations. + +### 6. Check what the index serves + +```bash +uvx --from ca-tariff-parse==X.Y.Z ca-tariff-parse --version +uvx --from ca-tariff-parse==X.Y.Z python -c \ + "from importlib.metadata import metadata as m; print(m('ca-tariff-parse').get_all('Project-URL'))" +``` + +Then change the README's *Quick start* to `uv tool install ca-tariff-parse`, +delete the sentence saying it is not on PyPI, and close Phase 8 in +`docs/ROADMAP.md`. + +## If something fails + +| Symptom | Cause | +| --- | --- | +| `publish` fails with a trusted-publisher error | Step 2 is missing or a field differs. Compare the error's claims with the table. | +| `verify` fails at `git verify-tag` | The tag is not signed by a key in `.github/allowed_signers`, or the tag is lightweight. | +| `verify` fails at `gh release view` | The release is still a draft (step 5.2). | +| PyPI rejects the upload as a duplicate file | That version exists already, or existed and was deleted. Use the next number. | +| `build` fails on `test "$(ls dist \| wc -l)" -eq 2` | Something else wrote into `dist/`. Look at the job's file listing. | + +## Known limit of the PyPI page + +The README links to repository files with relative paths such as +`docs/adr/...`. PyPI renders those as links that go nowhere. The project page's +*Documentation* link takes a reader to the repository. Rewriting the links as +absolute URLs at build time is possible with a hatch metadata hook and is not +done here. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a8f45c3..bfe24c1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -341,6 +341,11 @@ it wrong produces a failure that reads like a permissions problem rather than a typo. Until that registration exists, the package is not on PyPI, the publish job fails closed at the upload, and the README says so. +**Checked 2026-10-17.** The built wheel did not start when installed, so the +`v0.3.0` tag must not be uploaded; the fix, a clean-environment check +(`make dist-check`) and the publishing procedure are in +[RELEASING.md](RELEASING.md). The next release is the first that can go to PyPI. + **Done when.** A release is on PyPI under the project's name, or the README's install section records that the tag is the distribution and the workflow is removed. Either closes the phase honestly. diff --git a/pyproject.toml b/pyproject.toml index 9e28d0a..fda03b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ Homepage = "https://github.com/ChelseaKR/ca-tariff-parse" Source = "https://github.com/ChelseaKR/ca-tariff-parse" Issues = "https://github.com/ChelseaKR/ca-tariff-parse/issues" +Changelog = "https://github.com/ChelseaKR/ca-tariff-parse/blob/main/CHANGELOG.md" +Documentation = "https://github.com/ChelseaKR/ca-tariff-parse#readme" +Security = "https://github.com/ChelseaKR/ca-tariff-parse/blob/main/SECURITY.md" [project.scripts] ca-tariff-parse = "ca_tariff_parse.cli:main" @@ -46,6 +49,12 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/ca_tariff_parse"] +# The published schema lives at the repository root so it can be linked and +# versioned as a document. The installed `export` command also reads it, so the +# wheel carries a copy under the package. See tests/test_distribution.py. +[tool.hatch.build.targets.wheel.force-include] +"schemas/parsed-schedule-v1.schema.json" = "ca_tariff_parse/_schemas/parsed-schedule-v1.schema.json" + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q --strict-markers --strict-config --cov=ca_tariff_parse --cov-report=term-missing --cov-fail-under=85" diff --git a/scripts/dist-check.sh b/scripts/dist-check.sh new file mode 100755 index 0000000..6c7a627 --- /dev/null +++ b/scripts/dist-check.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Build the sdist and wheel, then install the wheel into an empty environment +# outside the checkout and run it. Nothing is uploaded. +# +# This is the check a release is owed before anyone presses publish: an index +# release is immutable, so a wheel that fails on import, or that lacks its +# Project-URL metadata, can only be corrected by spending a version number. +# +# make dist-check +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +cd "${root}" +uv build --out-dir "${work}/dist" +wheel="$(ls "${work}"/dist/*.whl)" +sdist="$(ls "${work}"/dist/*.tar.gz)" +test "$(ls "${work}/dist" | grep -c -E '\.(whl|tar\.gz)$')" -eq 2 + +# Metadata the index renders: long description, classifiers, license, URLs. +uvx --from 'twine==7.0.0' twine check --strict "${work}"/dist/* + +# The sdist has to build a wheel on its own, as a consumer of it would. +uv build --wheel --out-dir "${work}/from-sdist" "${sdist}" + +# A clean interpreter, a clean directory, no checkout on the path. +uv venv --quiet --python 3.12 "${work}/venv" +uv pip install --quiet --python "${work}/venv/bin/python" "${wheel}" +mkdir "${work}/run" +cd "${work}/run" +cli="${work}/venv/bin/ca-tariff-parse" +py="${work}/venv/bin/python" + +"${cli}" --version +"${cli}" --help > /dev/null +fixture="${root}/tests/fixtures/SYNTHETIC-example-schedule-complete.txt" +"${cli}" parse "${fixture}" --min-coverage 1.0 > "${work}/parsed.json" +"${cli}" export "${work}/parsed.json" --table charges > /dev/null + +"${py}" -I - <<'PY' +from importlib.metadata import metadata + +meta = metadata("ca-tariff-parse") +urls = {u.split(", ", 1)[0]: u.split(", ", 1)[1] for u in meta.get_all("Project-URL") or []} +required = {"Homepage", "Source", "Issues", "Changelog", "Documentation", "Security"} +missing = required - set(urls) +assert not missing, f"wheel metadata lacks Project-URL: {sorted(missing)}" +assert meta["License-Expression"] == "Apache-2.0", meta["License-Expression"] +print("Project-URL:", ", ".join(sorted(urls))) +PY +echo "dist-check: ok" diff --git a/src/ca_tariff_parse/export.py b/src/ca_tariff_parse/export.py index d57fec5..304211d 100644 --- a/src/ca_tariff_parse/export.py +++ b/src/ca_tariff_parse/export.py @@ -53,10 +53,17 @@ #: A row lifted out of its file still names the bytes it was read from. IDENTITY_COLUMNS = ("document_id", "document_sha256", "parser_version") -#: Where the schema lives relative to the installed package. Read at import -#: time from the repository checkout, and shipped beside the package when -#: installed. -SCHEMA_PATH = Path(__file__).resolve().parents[2] / "schemas" / "parsed-schedule-v1.schema.json" +_SCHEMA_NAME = "parsed-schedule-v1.schema.json" +_PACKAGE_SCHEMA = Path(__file__).resolve().parent / "_schemas" / _SCHEMA_NAME +_CHECKOUT_SCHEMA = Path(__file__).resolve().parents[2] / "schemas" / _SCHEMA_NAME + +#: Where the schema is read from, at import time. An installed wheel carries a +#: copy beside the package (``pyproject.toml`` force-includes it under +#: ``_schemas/``); a repository checkout, including an editable install, reads +#: the published file at the repository root, so there is one source of truth +#: and the copy cannot drift from it. Without the copy an installed +#: ``ca-tariff-parse`` failed on import, before it could print ``--help``. +SCHEMA_PATH = _PACKAGE_SCHEMA if _PACKAGE_SCHEMA.is_file() else _CHECKOUT_SCHEMA #: ``$defs`` names that hold a cited scalar rather than a record. CITED_DEFS = frozenset({"citedString", "citedStringOrNull"}) diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..29d724e --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,81 @@ +"""What `pip install ca-tariff-parse` would hand a reader. + +An index release is immutable, so a defect in the wheel cannot be fixed in +place: the next version number has to carry the fix. Two have already cost +another project in this family a release: a wheel with no `Project-URL` +metadata, and a wheel that could not find a file it read from the repository +root. These tests check the configuration that decides both. The built wheel +itself is installed into a clean environment and run by the `dist` job in +`.github/workflows/ci.yml`. +""" + +from __future__ import annotations + +import tomllib +from typing import Any +from urllib.parse import urlparse + +from ca_tariff_parse import export + +from .conftest import REPO_ROOT + +REPOSITORY = "https://github.com/ChelseaKR/ca-tariff-parse" +#: Labels a reader looks for on the project page. `Source` and `Issues` are +#: the ones a reviewer of the index listing checks first. +REQUIRED_URLS = ("Homepage", "Source", "Issues", "Changelog", "Documentation", "Security") + + +def _pyproject() -> dict[str, Any]: + data: dict[str, Any] = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + return data + + +def test_the_project_urls_are_declared_and_point_at_the_repository() -> None: + urls = _pyproject()["project"]["urls"] + for label in REQUIRED_URLS: + assert label in urls, f"[project.urls] has no {label}" + for label, url in urls.items(): + parsed = urlparse(url) + assert parsed.scheme == "https", label + assert ( + url == REPOSITORY + or url.startswith(f"{REPOSITORY}/") + or url.startswith(f"{REPOSITORY}#") + ), label + + +def test_the_project_urls_that_name_a_file_name_one_that_exists() -> None: + urls = _pyproject()["project"]["urls"] + for label in ("Changelog", "Security"): + path = urls[label].removeprefix(f"{REPOSITORY}/blob/main/") + assert (REPO_ROOT / path).is_file(), f"{label} points at {path}, which does not exist" + + +def test_the_index_listing_has_what_it_needs() -> None: + project = _pyproject()["project"] + assert project["readme"] == "README.md" + assert project["license"] + assert project["requires-python"] + assert project["classifiers"] + assert project["scripts"]["ca-tariff-parse"] == "ca_tariff_parse.cli:main" + + +def test_the_wheel_carries_the_schema_the_export_command_reads() -> None: + include = _pyproject()["tool"]["hatch"]["build"]["targets"]["wheel"]["force-include"] + assert include, "the wheel force-includes nothing" + (source, target), *rest = include.items() + assert not rest + assert (REPO_ROOT / source).is_file() + # Beside the package, where export.py looks first when installed. + assert target == "ca_tariff_parse/_schemas/" + source.rsplit("/", 1)[-1] + assert export._PACKAGE_SCHEMA.relative_to(export._PACKAGE_SCHEMA.parents[2]).as_posix() == ( + target + ) + + +def test_a_checkout_reads_the_one_published_schema() -> None: + # No second copy lives in the source tree: a copy could drift from the + # schema that docs and consumers link to. + assert not export._PACKAGE_SCHEMA.exists() + assert export.SCHEMA_PATH == export._CHECKOUT_SCHEMA + assert export.SCHEMA_PATH.is_file()