From 2ec38f27767a5bd49f65c46a825bd4d34438032f Mon Sep 17 00:00:00 2001 From: Oleg <18466855+EvaTheSalmon@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:08:45 +0300 Subject: [PATCH 1/2] feat(cli): collapse the surface to site and bench semibase is a setup script: one run brings an instance to a known state and exits. The old surface named the tool's internal steps (config, create, verify) plus a mechanism (all, whose only caller was this repository's CI). It is now two commands named for the two situations, and they differ in exactly one thing: site applies the ALTER SYSTEM memory constants, bench does not. config, create, verify and all are removed with no alias and no deprecation path. public.trends moves into this tool, in both modes. It was created by Simple-Scada 2 on a site and by SemiPlot's seeder on a bench, so the reader's access to it was unknowable until after the SCADA's first start. sql/trends.sql carries the vendor's shape verbatim - partitioned by range on t, primary key tpk, the tpdefault catch-all - embedded like sql/semiplot_tags.sql and applied over a scada_writer login of its own, never SET ROLE, so the reader's SELECT arrives through the default privileges set for that role. messages is not created and day partitions are not created; both stay with their writer. verify's two load-bearing checks become the tail of both commands, where they are now knowable: the reader holds SELECT on public.trends and does not hold INSERT, plus the pending-restart warning on site. The messages check, the "writer has not run yet" inference and the asPartOfAll branch go with the commands they served. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 16 +-- CLAUDE.md | 44 +++++--- README.md | 4 +- cmd/semibase/main.go | 28 ++--- cmd/semibase/main_test.go | 30 ++++-- docs/architecture/README.md | 13 +-- docs/architecture/configuration.md | 17 +-- docs/architecture/overview.md | 37 ++++--- docs/architecture/provisioning.md | 122 +++++++++++++++------- docs/deployment.md | 66 ++++++------ docs/readme.md | 2 +- embed.go | 3 + internal/provision/config.go | 2 +- internal/provision/provision.go | 148 +++++++++++++-------------- internal/provision/provision_test.go | 22 ++++ sql/trends.sql | 30 ++++++ 16 files changed, 359 insertions(+), 225 deletions(-) create mode 100644 sql/trends.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d97bbd..8d23e61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,10 +55,10 @@ jobs: - name: Test run: go test -race ./... - # The bench path: the same binary provisions the service container the - # way it provisions production. Run twice to prove idempotency. "all" - # ends with verify, which reports "writer has not run" as a state, not - # a failure - the expected order before any SCADA has started. + # The site path, the only thing that covers the ALTER SYSTEM tuning: the + # same binary provisions the service container the way it provisions an + # installation machine. Run twice to prove idempotency - the second run + # finds every object in place and still exits on a passing access check. - name: Provision the service container env: SEMIBASE_SUPER_PASSWORD: ci-super @@ -66,15 +66,15 @@ jobs: SEMIBASE_READER_PASSWORD: ci-reader run: | go build -o semibase ./cmd/semibase - ./semibase all --database semiplot_dev --expected-major 17 - ./semibase all --database semiplot_dev --expected-major 17 + ./semibase site --database semiplot_dev --expected-major 17 + ./semibase site --database semiplot_dev --expected-major 17 # The socket path, which nothing else covers: the same binary, run the way # the official image runs an init script. During init the server has # listen_addresses empty and only /var/run/postgresql answers, superuser # auth is local trust, and the script's own "set -e" makes the entrypoint # abort on failure. So the container reaching a state where semiplot_reader - # selects over TCP proves create completed over the socket. + # selects over TCP proves bench completed over the socket. - name: Provision over a unix socket, as a postgres image init script env: SEMIBASE_WRITER_PASSWORD: ci-writer @@ -96,7 +96,7 @@ jobs: cat > initdb-semibase.sh <<'SCRIPT' #!/bin/sh set -e - /semibase create --host /var/run/postgresql --database semiplot_dev --expected-major 17 + /semibase bench --host /var/run/postgresql --database semiplot_dev --expected-major 17 SCRIPT # the entrypoint sources a non-executable init script instead of running it, # and a binary without the bit set cannot run at all diff --git a/CLAUDE.md b/CLAUDE.md index 81a253b..0709c58 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,10 +6,10 @@ provisions and configures the instance that hosts the Simple-Scada 2 archive: th CLI binary, `semibase.exe`, a container image carrying its Linux build, plus the instance's architecture docs. Deployment target: Windows. Language: Go (module `github.com/Semiteq/SemiBase`), driver -`pgx/v5`, `sql/semiplot_tags.sql` embedded via `go:embed`. Entry point: `cmd/semibase`. -The module is pure `pgx` and compiles for any GOOS; Linux builds serve the containerised -test bench of consumers (SemiPlot runs `create` against an ephemeral `postgres:17` container). -All commands run from the repository root. +`pgx/v5`, `sql/semiplot_tags.sql` and `sql/trends.sql` embedded via `go:embed`. Entry point: +`cmd/semibase`. The module is pure `pgx` and compiles for any GOOS; Linux builds serve the +containerised test bench of consumers (SemiPlot runs `bench` against an ephemeral `postgres:17` +container). All commands run from the repository root. ## Build @@ -80,13 +80,15 @@ golangci-lint run golangci-lint is pinned to 2.12.2 (`winget install GolangCI.golangci-lint --version 2.12.2`); config in `.golangci.yml`. CI (`.github/workflows/ci.yml`) runs `go build`, `go test -race`, and the same lint on `windows-latest` and `ubuntu-latest` for every push and pull request; -the Linux job also provisions a `postgres:17-alpine` service container by running `all` -twice — the bench path and the idempotency check in one step — and builds the container image, -so a broken `Dockerfile` fails on the pull request rather than at tag time. CI pushes nothing. +the Linux job also provisions a `postgres:17-alpine` service container by running `site` +twice — the tuning path and the idempotency check in one step — provisions a second container +with `bench` as a `postgres` image init script over the unix socket, and builds the container +image, so a broken `Dockerfile` fails on the pull request rather than at tag time. CI pushes +nothing. Unit tests live beside the source (`cmd/semibase/*_test.go`, `internal/provision/*_test.go`), -table-driven. There are no database-touching tests; the `verify` command is the integration -check, run against a live server. +table-driven. There are no database-touching tests in the repository; the integration check is +the reader-access check both commands run at the tail, against a live server. ## Format @@ -97,24 +99,29 @@ gofmt -w . # run before presenting changes; gofmt is authoritative ## Run ```powershell -.\semibase.exe --help # commands: config | create | verify | all | version -.\semibase.exe create --port 15432 --database semiplot_dev --expected-major 17 -.\semibase.exe verify --reader-password # post-writer proof of the reader access chain +.\semibase.exe --help # commands: site | bench | version +.\semibase.exe site # an installation machine: tuning, then everything else +.\semibase.exe bench --port 15432 --database semiplot_dev --expected-major 17 .\semibase.exe version # print the build revision ``` +`site` and `bench` differ in one thing: `site` applies the `ALTER SYSTEM` memory constants, +`bench` does not. Both create the database, the roles, the grants, `semiplot_tags` and +`public.trends`, and both end by checking that `semiplot_reader` reads `public.trends` and +cannot write it. A failed check is a non-zero exit. + Passwords come from flags, env, or a `.env` file in the working directory (flag > env > `.env`; template `.env.example`): `SEMIBASE_SUPER_PASSWORD`, `SEMIBASE_WRITER_PASSWORD`, -`SEMIBASE_READER_PASSWORD`. -`verify` failing with "writer has not run" before the SCADA's first start is the expected -order, not a bug. +`SEMIBASE_READER_PASSWORD`. The writer password is needed to create `public.trends`, which the +tool does over a `scada_writer` login so the table's owner is the role the SCADA writes with. ## Layout ``` cmd/semibase/ CLI entry point: command dispatch, flags, usage text -internal/provision/ the phases: config (ALTER SYSTEM), create (db/roles/grants), verify -sql/ DDL for objects we own (semiplot_tags), embedded into the binary +internal/provision/ the phases behind the two commands: config (ALTER SYSTEM), create + (db/roles/grants/semiplot_tags/trends), check (reader access) +sql/ embedded DDL: semiplot_tags (ours) and trends (the vendor's shape) docs/ human docs in Russian (enter at docs/readme.md) docs/architecture/ agent-facing design docs in English docs/plans/ dated implementation plans @@ -129,3 +136,6 @@ docs/plans/ dated implementation plans in `docs/plans/completed/`. - The archive schema itself belongs to the SCADA and is documented in the SemiPlot repository (`docs/architecture/scada-archive.md` there); this repository documents the instance around it. + `sql/trends.sql` is a transcription of the vendor's shape, created once and never altered — + whether the SCADA tolerates finding it already there is an unverified assumption, stated with + its experiment in `docs/architecture/provisioning.md`. diff --git a/README.md b/README.md index efbfa8e..073b8b9 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,6 @@ SemiBase — разворачиваемый PostgreSQL-сервис для ус # установка pgsql winget install --id PostgreSQL.PostgreSQL.17 --exact -# конфигурация сервера, архивная база, роли, права -.\semibase.exe all +# конфигурация сервера, архивная база, роли, права, таблица архива +.\semibase.exe site ``` diff --git a/cmd/semibase/main.go b/cmd/semibase/main.go index 47aeffe..7c2bd5d 100644 --- a/cmd/semibase/main.go +++ b/cmd/semibase/main.go @@ -17,20 +17,22 @@ import ( "github.com/Semiteq/SemiBase/internal/provision" ) -const usage = `semibase - provisions the SemiBase PostgreSQL instance. +const usage = `semibase - brings a PostgreSQL instance to its provisioned state and exits. Usage: semibase [flags] Commands: - config apply the server configuration deltas (ALTER SYSTEM + reload); - shared_buffers takes effect at the next service restart - create create the archive database, the roles, the access chain, and semiplot_tags - verify post-writer checks: archive tables exist, the reader reads and cannot write - all config + create + verify + site an installation machine: apply the memory tuning (ALTER SYSTEM + reload), + then create the archive database, the roles, the access chain, + semiplot_tags and public.trends + bench a throwaway container: the same, without the tuning version print the build revision -Every step checks before it creates; re-running any command is safe. Passwords of +Both commands end by checking that semiplot_reader can read public.trends and +cannot write it; a failed check is a non-zero exit. + +Every step checks before it creates; re-running either command is safe. Passwords of existing roles change only when the corresponding flag or variable is set. Passwords come from flags, environment variables, or a .env file in the working @@ -55,10 +57,8 @@ var ( // only tests write to this map, to insert stubs var commands = map[string]func(context.Context, provision.Options) error{ - "config": func(ctx context.Context, options provision.Options) error { return options.Config(ctx) }, - "create": func(ctx context.Context, options provision.Options) error { return options.Create(ctx) }, - "verify": func(ctx context.Context, options provision.Options) error { return options.Verify(ctx) }, - "all": func(ctx context.Context, options provision.Options) error { return options.All(ctx) }, + "site": func(ctx context.Context, options provision.Options) error { return options.Site(ctx) }, + "bench": func(ctx context.Context, options provision.Options) error { return options.Bench(ctx) }, } func main() { @@ -141,8 +141,10 @@ func revisionFromSettings(settings []debug.BuildSetting) string { return vcsRevision } -// password flags default to empty so the usage output can only ever show the -// variable names, never their values +// One flag set serves both commands: site and bench differ only in the tuning phase, +// which reads no flag of its own, so every flag below is used by either command. +// Password flags default to empty so the usage output can only ever show the variable +// names, never their values. func newFlagSet(command string, options *provision.Options) *flag.FlagSet { flags := flag.NewFlagSet(command, flag.ContinueOnError) flags.SetOutput(errorOutput) diff --git a/cmd/semibase/main_test.go b/cmd/semibase/main_test.go index 3feee2e..8a36eb7 100644 --- a/cmd/semibase/main_test.go +++ b/cmd/semibase/main_test.go @@ -66,7 +66,7 @@ func TestFlagUsageHidesPasswordValues(t *testing.T) { } options := provision.Options{} - flags := newFlagSet("create", &options) + flags := newFlagSet("bench", &options) buffer := &bytes.Buffer{} flags.SetOutput(buffer) flags.PrintDefaults() @@ -134,7 +134,7 @@ func TestParseOptions(t *testing.T) { t.Setenv(name, value) } - options, err := parseOptions("create", tt.arguments) + options, err := parseOptions("bench", tt.arguments) switch { case tt.wantErr != nil: @@ -221,6 +221,22 @@ func TestRunVersionCommandExitsZero(t *testing.T) { } } +// the surface is two commands and nothing else: config, create, verify and all were +// removed with no alias and no tombstone +func TestCommandSurface(t *testing.T) { + want := map[string]bool{"site": true, "bench": true} + for name := range commands { + if !want[name] { + t.Errorf("unexpected command %q in the dispatch map", name) + } + } + for name := range want { + if _, known := commands[name]; !known { + t.Errorf("command %q is missing from the dispatch map", name) + } + } +} + func TestRunUnmappedCommandIsAnError(t *testing.T) { buffer := captureErrorOutput(t) @@ -247,8 +263,8 @@ func TestRunHelpFlagExitsZero(t *testing.T) { buffer := captureErrorOutput(t) clearPasswordEnvironment(t) - if got := run(context.Background(), []string{"create", "--help"}); got != 0 { - t.Fatalf("run(create --help) = %d, want exit code 0", got) + if got := run(context.Background(), []string{"bench", "--help"}); got != 0 { + t.Fatalf("run(bench --help) = %d, want exit code 0", got) } if !strings.Contains(buffer.String(), "-super-password") { t.Errorf("help output does not list the command flags: %q", buffer.String()) @@ -336,13 +352,13 @@ func TestCancelledContextReturnsPromptly(t *testing.T) { cancel() started := time.Now() - err := commands["create"](ctx, options) + err := commands["bench"](ctx, options) elapsed := time.Since(started) if !errors.Is(err, context.Canceled) { - t.Fatalf("create with cancelled context returned %v, want context.Canceled in the chain", err) + t.Fatalf("bench with cancelled context returned %v, want context.Canceled in the chain", err) } if elapsed > 2*time.Second { - t.Errorf("create with cancelled context took %v, want a prompt return", elapsed) + t.Errorf("bench with cancelled context took %v, want a prompt return", elapsed) } } diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 1ba3ceb..6905cef 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -8,8 +8,8 @@ Declarative architecture docs (English, present tense). These describe the syste - [overview.md](./overview.md) — purpose, consumers, components, provisioning order. - [configuration.md](./configuration.md) — every setting changed from the PostgreSQL default, with why. -- [provisioning.md](./provisioning.md) — the tool's commands, idempotency rules, and the - reader-access chain. +- [provisioning.md](./provisioning.md) — the tool's two commands, idempotency rules, the + reader-access chain, and the stated assumption about the SCADA meeting an existing `trends`. ## Locked decisions @@ -18,12 +18,13 @@ Declarative architecture docs (English, present tense). These describe the syste | Installation | PostgreSQL via `winget`, major version pinned, automatic upgrade disabled; engine install stays outside the tool | | Major version | 17 for new installs, pinned latest; floor 14 (SemiPlot needs `date_bin`; Simple-Scada 2 docs require 14+) | | Provisioning tool | Go single binary `semibase.exe` (`pgx/v5`, embedded SQL); no runtime or `psql` on the target machine | -| Instance configuration | Fixed `ALTER SYSTEM` deltas applied by `semibase config` via `pg_reload_conf()`, never by editing `postgresql.conf`; `shared_buffers` takes effect at the next service restart, reported through `pg_settings.pending_restart` | +| Instance configuration | Fixed `ALTER SYSTEM` deltas applied by `semibase site` via `pg_reload_conf()`, never by editing `postgresql.conf`; `shared_buffers` takes effect at the next service restart, reported through `pg_settings.pending_restart` | | Hardware floor | Installation machines guarantee 8 GB RAM and 50 GB database disk; memory settings are fixed constants sized to that floor, identical on every machine | | Roles | `scada_writer` (SCADA), `semiplot_reader` (viewers, `SELECT` only); `semiplot_*` objects owned by `postgres` — a dedicated owner role appears when a tag-editing mechanism exists | -| Reader access | `ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer` set **before** the writer first runs | -| Objects we add | `semiplot_tags` only — no triggers, functions, scheduled jobs, or extensions | -| Archive schema | Owned by Simple-Scada 2; documented in the SemiPlot repository, never created or altered here | +| Command surface | Two commands, `site` and `bench`, named for the situation rather than for the tool's internal steps; they differ in one thing, the memory tuning, which only `site` applies | +| Reader access | `ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer` set **before** `public.trends` is created, and before the writer first runs | +| Objects we add | `semiplot_tags` and the vendor-shaped `public.trends` — no triggers, functions, scheduled jobs, or extensions; `messages` is not created, nothing we ship reads it | +| Archive schema | Owned by Simple-Scada 2 and documented in the SemiPlot repository. This tool creates `public.trends` once, with the vendor's shape, connected as `scada_writer`, and never alters it afterwards; day partitions and every later change to the schema remain the vendor's | | Distribution | Binaries as GitHub release assets; the Linux binary also as `ghcr.io/semiteq/semibase`, `FROM scratch`, one file, pushed after the release exists; benches track `:latest` and a prerelease never moves it | | Backup method | `UNDECIDED` — commissioning-time, needs the customer's archive size | | Retention depth / disk sizing | `UNDECIDED` — needs a measured write rate from a working installation | diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index 3e8f0b7..b997358 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -1,6 +1,6 @@ # Instance configuration -Every setting the `config` command writes is listed. Most differ from the PostgreSQL default; +Every setting the `site` command writes is listed. Most differ from the PostgreSQL default; two (`effective_cache_size`, `checkpoint_completion_target`) equal it and are pinned deliberately, so the values in force are visible here rather than implied. The workload is a sustained append-only insert stream from one writer plus a small number of read-heavy @@ -10,12 +10,15 @@ Every installation machine guarantees at least 8 GB of RAM and 50 GB of disk for so the memory figures are fixed constants sized to that floor: every machine runs the identical configuration, and no setting depends on inspecting the host. -The `config` command of `semibase.exe` writes these through `ALTER SYSTEM`, so they live in -`postgresql.auto.conf` and survive a reinstall of `postgresql.conf`, and applies them with -`pg_reload_conf()`. Every setting below except `shared_buffers` is reload-context or weaker and -takes effect immediately; `shared_buffers` takes effect at the next service restart or reboot. -`config` prints the server's own pending-restart list (`pg_settings.pending_restart`), and -`verify` warns while any setting still waits, so a missed restart surfaces during commissioning. +`semibase site` writes these through `ALTER SYSTEM`, so they live in `postgresql.auto.conf` and +survive a reinstall of `postgresql.conf`, and applies them with `pg_reload_conf()`. Every setting +below except `shared_buffers` is reload-context or weaker and takes effect immediately; +`shared_buffers` takes effect at the next service restart or reboot. The tuning phase prints the +server's own pending-restart list (`pg_settings.pending_restart`) and `site` warns about it again +on its last lines, so a missed restart surfaces during commissioning. + +`semibase bench` writes none of them: the values are constants sized to the installation +machine's hardware floor and mean nothing to a throwaway container. ## Server settings diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index f969860..4eba938 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -15,8 +15,8 @@ its own retention setting. | Consumer | Role | Access | | --- | --- | --- | -| Simple-Scada 2 (archives into PostgreSQL) | `scada_writer` | Creates and writes `trends`/`messages` and their partitions; executes retention | -| SemiPlot (trend viewer) | `semiplot_reader` | `SELECT` on `trends`, `messages`, `semiplot_tags` — nothing else | +| Simple-Scada 2 (archives into PostgreSQL) | `scada_writer` | Owns and writes `trends`, creates its day partitions, creates and writes `messages`; executes retention | +| SemiPlot (trend viewer) | `semiplot_reader` | `SELECT` on `trends` and `semiplot_tags` — nothing else | The `semiplot_*` objects are owned by `postgres`; commissioning fills `semiplot_tags` as the superuser. A dedicated owner role appears when a tag-editing mechanism exists to hold it. @@ -28,8 +28,9 @@ plaintext password in a client configuration file an acceptable risk. | Component | Responsibility | | --- | --- | -| `cmd/semibase`, `internal/provision` | `semibase.exe` — idempotent provisioning: instance configuration, database, roles, grants, `semiplot_tags`, post-writer verification | -| `sql/semiplot_tags.sql` | DDL for the one object we add to the archive database, embedded into the binary | +| `cmd/semibase`, `internal/provision` | `semibase.exe` — idempotent provisioning in two commands, `site` and `bench`: instance tuning (`site` only), database, roles, grants, `semiplot_tags`, `public.trends`, and the reader-access checks that end every run | +| `sql/semiplot_tags.sql` | DDL for the one object we invented, embedded into the binary | +| `sql/trends.sql` | The vendor's archive-table shape, transcribed and embedded; applied as `scada_writer` | | `Dockerfile` | `ghcr.io/semiteq/semibase` — the Linux binary alone on `scratch`, for consumers that layer it into a bench image | | `docs/architecture/` | The instance as it is: configuration deltas, provisioning order, ownership | @@ -40,20 +41,24 @@ silently stop. ## Provisioning order -The order matters, because the archive tables do not exist until the SCADA has run once. +The order matters, because the default privileges have to be in place before any table +`scada_writer` owns is created. 1. Install the engine: `winget install --id PostgreSQL.PostgreSQL.17 --exact`. -2. `semibase config` — apply the configuration deltas; `shared_buffers` takes effect at the - next service restart or reboot, and `verify` warns while it waits. -3. `semibase create` — archive database, both roles, default privileges, `semiplot_tags`. -4. Point the Simple-Scada project at the database and start it once. It creates `trends`, - `messages` and the first daily partitions. -5. `semibase verify` — prove the reader access chain against the tables the writer created. -6. Fill `semiplot_tags` with the variables to be trended. -7. Write the SemiPlot connection file, including the source time zone. - -Every consumer must survive every intermediate state of this sequence — no database, database -without `trends`, `trends` without `semiplot_tags`, `semiplot_tags` without rows. Each is a normal +2. `semibase site` — tuning, then the archive database, both roles, default privileges, + `semiplot_tags` and `public.trends`; it ends by proving the reader reads `trends` and cannot + write it, and warns while `shared_buffers` waits for a service restart. +3. Restart the PostgreSQL service or reboot the machine, so `shared_buffers` takes effect. +4. Point the Simple-Scada project at the database and start it once. It finds `trends` in place + and creates the day partitions and `messages` itself. +5. Fill `semiplot_tags` with the variables to be trended. +6. Write the SemiPlot connection file, including the source time zone. + +A bench replaces step 2 with `semibase bench` and has no step 3 or 4: the consumer's seeder plays +the writer's part. + +Every consumer must survive every intermediate state of this sequence — no database, empty +`trends`, `trends` without `semiplot_tags`, `semiplot_tags` without rows. Each is a normal condition, not a crash. ## Upgrades diff --git a/docs/architecture/provisioning.md b/docs/architecture/provisioning.md index 9e6556b..0b9ddd3 100644 --- a/docs/architecture/provisioning.md +++ b/docs/architecture/provisioning.md @@ -7,8 +7,9 @@ matters, is a liability. The binary is self-contained: `pgx/v5` speaks the wire protocol itself, over TCP or a unix socket, so neither `psql.exe` nor any runtime needs to be present on the target machine. -`sql/semiplot_tags.sql` is embedded at build time. Installing the PostgreSQL engine itself is one documented `winget` -line, deliberately outside the tool — OS package management is not database provisioning. +`sql/semiplot_tags.sql` and `sql/trends.sql` are embedded at build time. Installing the PostgreSQL +engine itself is one documented `winget` line, deliberately outside the tool — OS package management +is not database provisioning. ## Distribution @@ -69,12 +70,25 @@ no say in it. ## Commands +The tool is a setup script: it brings an instance to a known state and exits. It is run once, at +commissioning, and nobody comes back to run a second command by hand. So the surface names the two +situations it is run in, not the phases it runs through. + | Command | What it does | When | | --- | --- | --- | -| `config` | Writes the `ALTER SYSTEM` deltas from `configuration.md` and applies them with `pg_reload_conf()`; prints the server's pending-restart list (`shared_buffers` is the one entry, active after the next service restart or reboot) | Production. Never against a shared dev server — it retunes the whole instance; a throwaway bench container needs no tuning | -| `create` | Creates the archive database, the roles, the grants, the default privileges, and `semiplot_tags` | Before the SCADA's first start | -| `verify` | Proves the reader access chain against the tables the writer created | After the SCADA's first start | -| `all` | `config` + `create` + `verify`; `verify` reports "writer has not run" as a distinct state, not a failure | Fresh instance | +| `site` | The memory tuning, then everything else | An installation machine | +| `bench` | Everything else, without the tuning | A throwaway container: a consumer's test bench or a development database | + +The two differ in exactly one thing, the tuning. The rest is identical, which is what makes a bench +a test of what a site runs: + +| Step | `site` | `bench` | +| --- | --- | --- | +| `ALTER SYSTEM` constants from `configuration.md`, applied with `pg_reload_conf()` | yes | no — the constants are sized for an installation machine and mean nothing to a container | +| Archive database, both roles, grants, default privileges, `semiplot_tags` | yes | yes | +| `public.trends` with the `tpdefault` partition, created as `scada_writer` | yes | yes | +| Reader-access checks at the tail | yes | yes | +| Pending-restart warning at the tail | yes | no — nothing was tuned, so nothing can be pending | `version` (or `--version`) prints the build revision: the ldflags value (`-X main.revision=...`) when a release sets it, otherwise the VCS revision from @@ -104,7 +118,7 @@ pgx then dials `/.s.PGSQL.` on the `unix` network. Widening the would be a trap, not a courtesy: libpq also accepts a leading `@` for Linux's abstract namespace, but pgx resolves such a host as a TCP name, so `semibase` leaves it on the TCP path. -The socket form is what lets `create` run as an init script in the official `postgres` image, +The socket form is what lets `bench` run as an init script in the official `postgres` image, whose entrypoint serves `/docker-entrypoint-initdb.d/` from a temporary server with `listen_addresses` set to empty — reachable over the socket only. The Linux CI job runs exactly that: it layers the freshly built binary onto `postgres:17-alpine` behind such an init script and @@ -120,7 +134,7 @@ never exist. The package that runs the SQL owns its guards; the CLI is a thin dispatcher. - **Database-name validation.** `Options.Validate()` checks `Database` against - `^[a-z_][a-z0-9_]*$` and runs first in every phase (`Config`, `Create`, `verify`). The + `^[a-z_][a-z0-9_]*$` and runs first in every phase (`config`, `create`, `check`). The pattern lives only in `internal/provision`; the CLI calls the same method for the friendly early error. - **Identifier quoting.** Every interpolation of the database name into DDL goes through @@ -133,60 +147,92 @@ The package that runs the SQL owns its guards; the CLI is a thin dispatcher. in-flight query through pgx instead of killing the process mid-DDL. The context itself — `signal.NotifyContext(..., os.Interrupt)` plus the five-minute per-command timeout (`phaseTimeout`) — is owned by the CLI dispatcher in `cmd/semibase`. -- **Pending-restart reporting.** The tool never touches the service manager. `config` applies - its deltas with `pg_reload_conf()` and reads `pg_settings.pending_restart` back from the - server; `verify` warns while any setting still waits. The operator restarts the service or - reboots the machine — the one manual step, and the tool says so instead of doing it. +- **Pending-restart reporting.** The tool never touches the service manager. The tuning phase + applies its deltas with `pg_reload_conf()` and reads `pg_settings.pending_restart` back from + the server; `site` reads it again at the tail, so the last thing an operator sees names a + pending `shared_buffers`. The operator restarts the service or reboots the machine — the one + manual step, and the tool says so instead of doing it. ## The reader-access chain -`semiplot_reader` needs `SELECT` on `trends` and `messages` — tables that do not exist until the -SCADA has run once. A plain `GRANT` cannot be issued on a table that is not there, so `create` -sets: +`semiplot_reader` needs `SELECT` on `trends` and on every day partition the SCADA creates later — +objects that do not exist when the grant has to be decided. A plain `GRANT` cannot be issued on a +table that is not there, so the create phase sets: ```sql ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer IN SCHEMA public GRANT SELECT ON TABLES TO semiplot_reader; ``` -This must be in place **before** the writer first starts; tables the writer creates afterwards are -readable automatically. If the writer ran first, the repair is a one-time -`GRANT SELECT ON ALL TABLES IN SCHEMA public` plus the default-privileges statement for the -partitions still to come — `verify` detects this state and says so. +This is in place **before** `public.trends` is created and before the writer first starts; tables +`scada_writer` creates afterwards are readable automatically. If a table was created before the +statement ran, the repair is a one-time `GRANT SELECT ON ALL TABLES IN SCHEMA public` plus the +default-privileges statement for the partitions still to come — the tail check detects that state +and prints the repair. On PostgreSQL 15 and later the `public` schema no longer grants `CREATE` to everyone, so `create` grants it to `scada_writer` explicitly. On 14 the grant is redundant and harmless. -## What `verify` proves +## The archive table + +`public.trends` is created by this tool, by both commands, from `sql/trends.sql` — the vendor's +shape, transcribed from a customer archive dump. Three properties of how it is created carry the +weight: + +- **The role, more than the shape.** The table is created over a `scada_writer` login of its own, + never with `SET ROLE`, because the reader's `SELECT` has to arrive through the default privileges + set for that role a moment earlier. A superuser-owned table would give the reader access for a + different reason than a site gets it, and a bench would then test a different thing from + production. A run that finds no `public.trends` and no writer password fails and says so. +- **`messages` is not created.** Nothing we ship reads it, and every object we create is a surface + that can drift from the vendor. +- **Day partitions are not created.** `tpYYYYmMMdDD` belongs to the SCADA on a site and to the + consumer's seeder on a bench. Only the `tpdefault` catch-all is created, so a row written while + no day partition exists lands somewhere instead of failing. + +Existence is checked first (`to_regclass('public.trends')`), so a second run leaves the table +untouched — including a table the SCADA has since altered. + +## Assumption: the SCADA meeting an existing `trends` + +**Unverified.** Simple-Scada 2's behaviour when its archive target already exists is undocumented +and unmeasured. The expectation is that it writes into the existing table the way it does after a +reconnect, because the shape it finds is the shape it makes. + +The experiment that settles it: set `log_statement = 'all'`, start a Simple-Scada project once +against a database provisioned by `semibase site`, and read from the PostgreSQL log the DDL the +SCADA issues. If it creates the table unconditionally, or alters what it finds, this section is +replaced by what was observed and `sql/trends.sql` is reconsidered. + +## What the tail checks prove + +Both commands end with them, and a failure is a non-zero exit — the run did not reach the state it +promises. They are knowable at exit precisely because this tool creates `public.trends` itself: -1. `public.trends` and `public.messages` exist — the writer has run. -2. `semiplot_reader` holds `SELECT` and does not hold `INSERT` on `trends`. -3. When the reader password is supplied, an actual connection as `semiplot_reader` reads one row — - the whole chain, not just the catalog view of it. -4. The default partition is empty. Rows in it mean a day partition was missing at write time, - which is a writer-side fault worth surfacing during commissioning. -5. No setting waits for a service restart (`pg_settings.pending_restart`) — a warning, not a - failure, so a pending `shared_buffers` never blocks the access-chain checks. +1. `semiplot_reader` holds `SELECT` on `public.trends`. Failing this prints the one repair message + that turns a silent misconfiguration into an actionable one. +2. `semiplot_reader` does not hold `INSERT` on `public.trends` — the reader is read-only. +3. `site` only: no setting waits for a service restart (`pg_settings.pending_restart`) — a warning, + not a failure, so a pending `shared_buffers` never blocks the access checks. ## Development benches The bench is an ephemeral vanilla `postgres:17-alpine` container. The consumer's test fixture -starts it and provisions it by running `create --database semiplot_dev --expected-major 17` — -the same code path production takes, which is what keeps the fixture from drifting into a -hand-written copy of the grants. `config` is not part of the bench: a throwaway container has -nothing to tune. 14 remains the floor `create` accepts through `--expected-major`; the bench -runs the pinned major, 17. +starts it and provisions it by running `bench --database semiplot_dev --expected-major 17` — the +same code path a site takes minus the tuning, which is what keeps the fixture from drifting into a +hand-written copy of the grants. 14 remains the floor accepted through `--expected-major`; the +bench runs the pinned major, 17. Two rules make the bench exercise what production exercises: -- The bench seeder connects **as `scada_writer`** and creates the archive tables itself, the way - the SCADA would. This is what makes the default-privileges chain a daily-tested path instead of - a commissioning-day surprise. +- `public.trends` is created **as `scada_writer`**, by both commands, so the default-privileges + chain is a daily-tested path instead of a commissioning-day surprise. - Bench integration tests read **as `semiplot_reader`**, never as a superuser, so a broken grant fails a test today. -The seeder and the tests live in the SemiPlot repository; this repository only promises them a -correctly provisioned database. +The consumer's seeder fills the archive and creates the day partitions its data needs; it and the +tests live in the SemiPlot repository. This repository promises them a provisioned database with an +empty `public.trends` in it. ## Passwords diff --git a/docs/deployment.md b/docs/deployment.md index 5570914..6c43821 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,15 +1,15 @@ # Развёртывание -SemiBase разворачивается одной утилитой — `semibase.exe`. Утилита идемпотентна: каждый шаг -сначала проверяет состояние, затем создаёт недостающее. Повторный запуск любой команды -безопасен. Пароли существующих ролей меняются только при явной передаче соответствующего -флага или переменной окружения. +SemiBase разворачивается одной утилитой — `semibase.exe`. Это скрипт установки: одна команда +приводит экземпляр в заданное состояние и завершается. Утилита идемпотентна: каждый шаг сначала +проверяет состояние, затем создаёт недостающее. Повторный запуск безопасен. Пароли существующих +ролей меняются только при явной передаче соответствующего флага или переменной окружения. Справка: ```powershell .\semibase.exe --help # список команд -.\semibase.exe create --help # флаги конкретной команды +.\semibase.exe site --help # флаги конкретной команды ``` ## Требования @@ -29,13 +29,14 @@ SemiBase разворачивается одной утилитой — `semibas ## Команды -| Команда | Действие | -| --------- | --------------------------------------------------------------------------------------------------------------------------- | -| `config` | Настройки сервера через `ALTER SYSTEM` с немедленным применением; `shared_buffers` вступает в силу после перезапуска службы | -| `create` | Архивная база, роли, права доступа, таблица `semiplot_tags` | -| `verify` | Проверка после первого запуска SCADA: таблицы существуют, читатель читает и не пишет | -| `all` | `config` + `create` + `verify` | -| `version` | Печать ревизии сборки | +| Команда | Действие | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `site` | Машина установки: настройки сервера через `ALTER SYSTEM`, затем архивная база, роли, права доступа, таблицы `semiplot_tags` и `public.trends` | +| `bench` | Временный контейнер (тестовый стенд, база для разработки): то же самое без настроек сервера — константы памяти рассчитаны на машину установки и контейнеру ничего не дают | +| `version` | Печать ревизии сборки | + +Обе команды заканчиваются проверкой: `semiplot_reader` читает `public.trends` и не может в неё +писать. Ненулевой код возврата означает, что заявленное состояние не достигнуто. При обращении за поддержкой приложите вывод `semibase version` — он однозначно называет сборку утилиты на машине. @@ -56,16 +57,18 @@ SemiBase разворачивается одной утилитой — `semibas | Роль | Кто использует | Доступ | | ----------------- | -------------------------- | --------------------------------------------------- | -| `scada_writer` | Simple-Scada 2 | Создаёт и пишет архивные таблицы, выполняет ретеншн | -| `semiplot_reader` | SemiPlot и другие читатели | Только чтение `trends`, `messages`, `semiplot_tags` | +| `scada_writer` | Simple-Scada 2 | Владеет архивными таблицами и пишет в них, создаёт дневные партиции, выполняет ретеншн | +| `semiplot_reader` | SemiPlot и другие читатели | Только чтение `trends` и `semiplot_tags` | Объекты `semiplot_*` принадлежат `postgres`; заполнение `semiplot_tags` на пусконаладке -выполняется от суперпользователя. +выполняется от суперпользователя. Таблицу `public.trends` утилита создаёт от имени +`scada_writer` — владельцем должна быть та же роль, от которой пишет SCADA, иначе читатель +получит доступ не тем путём, каким он работает в эксплуатации. ## Порядок ввода в эксплуатацию -Порядок важен: архивные таблицы появляются только после первого запуска SCADA, а права -на чтение для них выставляются заранее, командой `create`. +Порядок важен: права на чтение выставляются до создания `public.trends`, поэтому утилита +отрабатывает до первого запуска SCADA. 1. Установить СУБД: @@ -75,32 +78,31 @@ winget install --id PostgreSQL.PostgreSQL.17 --exact 2. Задать пароли ролей: скопировать `.env.example` в `.env` рядом с утилитой и заполнить. -3. Настроить сервер и создать базу, роли и права: +3. Выполнить одну команду: ```powershell -.\semibase.exe config -.\semibase.exe create +.\semibase.exe site ``` -Все настройки, кроме `shared_buffers`, применяются сразу; `shared_buffers` — после -перезапуска службы PostgreSQL или перезагрузки машины. Перезагрузка обязательна до сдачи; -`verify` предупреждает, пока она не сделана. - -4. Запустить проект Simple-Scada против базы один раз — он создаёт `trends`, `messages` - и первые дневные партиции. +На выходе утилита гарантирует: настройки сервера применены, архивная база, обе роли, права +и права по умолчанию созданы, `semiplot_tags` и `public.trends` (с партицией `tpdefault`) +на месте, `semiplot_reader` читает `public.trends` и не может в неё писать. Отдельной +проверки после этого не требуется — она уже выполнена, её результат в коде возврата. -5. Проверить цепочку доступа читателя: +4. Перезапустить службу PostgreSQL или перезагрузить машину. Все настройки, кроме + `shared_buffers`, применяются сразу; `shared_buffers` — только после перезапуска. + Утилита предупреждает об этом в последних строках вывода, пока перезапуск не сделан. -```powershell -.\semibase.exe verify -``` +5. Запустить проект Simple-Scada против базы. Таблицу `trends` он застаёт готовой и дальше + создаёт дневные партиции и `messages` сам. 6. Заполнить `semiplot_tags` перечнем переменных для трендов. 7. Записать файл подключения SemiPlot, включая часовой пояс источника. -Если `verify` сообщает, что читатель не имеет доступа к `trends`, значит SCADA была -запущена раньше команды `create`. Способ починки утилита называет в тексте ошибки. +Если проверка в конце сообщает, что `semiplot_reader` не имеет доступа к `trends`, значит +таблица была создана раньше, чем выставлены права по умолчанию. Способ починки утилита +называет в тексте ошибки. ## Основные флаги diff --git a/docs/readme.md b/docs/readme.md index 60bf81b..5d1b1b5 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,4 +1,4 @@ # Документация SemiBase -- [Развёртывание](./deployment.md) — требования, установка СУБД, фазы скрипта, параметры, +- [Развёртывание](./deployment.md) — требования, установка СУБД, команды утилиты, параметры, порядок ввода в эксплуатацию. diff --git a/embed.go b/embed.go index 4def537..08ed30e 100644 --- a/embed.go +++ b/embed.go @@ -4,3 +4,6 @@ import _ "embed" //go:embed sql/semiplot_tags.sql var SemiplotTagsSQL string + +//go:embed sql/trends.sql +var TrendsSQL string diff --git a/internal/provision/config.go b/internal/provision/config.go index 21026d9..cdc2ec0 100644 --- a/internal/provision/config.go +++ b/internal/provision/config.go @@ -27,7 +27,7 @@ var settings = []setting{ {"track_io_timing", "on"}, } -func (o Options) Config(ctx context.Context) error { +func (o Options) config(ctx context.Context) error { if err := o.Validate(); err != nil { return err } diff --git a/internal/provision/provision.go b/internal/provision/provision.go index ff7b509..dd84bd6 100644 --- a/internal/provision/provision.go +++ b/internal/provision/provision.go @@ -41,18 +41,32 @@ func (o Options) Validate() error { return nil } -func (o Options) All(ctx context.Context) error { - if err := o.Config(ctx); err != nil { +// the tuning phase is the only thing that can leave a setting waiting for a service +// restart, so only the site run reads pending_restart back at the end +const ( + afterTuning = true + withoutTuning = false +) + +// Site brings an installation machine to its commissioned state: memory tuning, then +// everything the archive needs. +func (o Options) Site(ctx context.Context) error { + if err := o.config(ctx); err != nil { return err } - if err := o.Create(ctx); err != nil { + if err := o.create(ctx); err != nil { return err } - return o.verify(ctx, true) + return o.check(ctx, afterTuning) } -func (o Options) Verify(ctx context.Context) error { - return o.verify(ctx, false) +// Bench brings a throwaway container to the same state minus the tuning: its memory +// constants are sized for an installation machine and mean nothing to a container. +func (o Options) Bench(ctx context.Context) error { + if err := o.create(ctx); err != nil { + return err + } + return o.check(ctx, withoutTuning) } // pgx dials a unix socket exactly when pgconn.isAbsolutePath accepts the host. Widening this @@ -139,11 +153,11 @@ func alterRolePasswordStatement(name, password string) string { return fmt.Sprintf("ALTER ROLE %s PASSWORD '%s'", name, escapeLiteral(password)) } -func (o Options) Create(ctx context.Context) error { +func (o Options) create(ctx context.Context) error { if err := o.Validate(); err != nil { return err } - step("Phase create: database %s, roles, access chain", o.Database) + step("Phase create: database %s, roles, access chain, archive table", o.Database) super, err := o.connect(ctx, "postgres", o.SuperUser, o.SuperPassword) if err != nil { @@ -200,8 +214,8 @@ func (o Options) Create(ctx context.Context) error { return fmt.Errorf("granting CREATE on schema public: %w", err) } - // trends/messages do not exist until the SCADA runs, so SELECT on them can only be - // granted ahead of time, through default privileges of their future owner + // the writer creates public.trends below and its partitions later, so SELECT on them + // can only be granted ahead of time, through default privileges of their owner defaultPrivileges := fmt.Sprintf( "ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA public GRANT SELECT ON TABLES TO %s", WriterRole, ReaderRole) @@ -217,6 +231,39 @@ func (o Options) Create(ctx context.Context) error { return fmt.Errorf("granting SELECT on semiplot_tags: %w", err) } ok("semiplot_tags in place, readable by %s", ReaderRole) + + return o.ensureArchiveTable(ctx, archive) +} + +// public.trends is created over a scada_writer login of its own, never with SET ROLE: the +// reader's SELECT has to arrive through the default privileges set for that role above, which +// is how a site gets it. A superuser-owned table would hand the reader access for a different +// reason, and a bench would then test something production does not do. +func (o Options) ensureArchiveTable(ctx context.Context, archive *pgx.Conn) error { + var exists bool + if err := archive.QueryRow(ctx, + "SELECT to_regclass('public.trends') IS NOT NULL").Scan(&exists); err != nil { + return fmt.Errorf("checking public.trends: %w", err) + } + if exists { + ok("public.trends exists, left untouched") + return nil + } + if o.WriterPassword == "" { + return fmt.Errorf("public.trends does not exist and no %s password was given to create it "+ + "as that role; supply --writer-password or SEMIBASE_WRITER_PASSWORD", WriterRole) + } + + writer, err := o.connect(ctx, o.Database, WriterRole, o.WriterPassword) + if err != nil { + return err + } + defer writer.Close(ctx) + + if _, err := writer.Exec(ctx, semibase.TrendsSQL); err != nil { + return fmt.Errorf("applying trends.sql as %s: %w", WriterRole, err) + } + ok("public.trends created by %s, partitioned by t, with the tpdefault partition", WriterRole) return nil } @@ -278,11 +325,13 @@ func (o Options) ensureDatabase(ctx context.Context, conn *pgx.Conn) error { return nil } -func (o Options) verify(ctx context.Context, asPartOfAll bool) error { +// the tail of every run. The tool created public.trends itself a moment ago, so the reader's +// access to it is knowable at exit instead of after the SCADA's first start. +func (o Options) check(ctx context.Context, reportPendingRestart bool) error { if err := o.Validate(); err != nil { return err } - step("Phase verify: post-writer checks") + step("Phase check: the reader access chain") archive, err := o.connect(ctx, o.Database, o.SuperUser, o.SuperPassword) if err != nil { @@ -290,38 +339,15 @@ func (o Options) verify(ctx context.Context, asPartOfAll bool) error { } defer archive.Close(ctx) - pending, err := pendingRestartSettings(ctx, archive) - if err != nil { - return err - } - if len(pending) > 0 { - warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", - strings.Join(pending, ", ")) - } - - var trendsExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.trends') IS NOT NULL").Scan(&trendsExists); err != nil { - return fmt.Errorf("checking public.trends: %w", err) - } - if !trendsExists { - message := "writer has not run yet: public.trends does not exist. " + - "Start the Simple-Scada project against this database once, then run verify." - if asPartOfAll { - note("%s", message) - return nil + if reportPendingRestart { + pending, err := pendingRestartSettings(ctx, archive) + if err != nil { + return err + } + if len(pending) > 0 { + warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", + strings.Join(pending, ", ")) } - return fmt.Errorf("%s", message) - } - ok("public.trends exists") - - var messagesExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.messages') IS NOT NULL").Scan(&messagesExists); err != nil { - return fmt.Errorf("checking public.messages: %w", err) - } - if messagesExists { - ok("public.messages exists") - } else { - warn("public.messages does not exist") } var canSelect bool @@ -330,8 +356,8 @@ func (o Options) verify(ctx context.Context, asPartOfAll bool) error { return fmt.Errorf("checking reader SELECT privilege: %w", err) } if !canSelect { - return fmt.Errorf("%s cannot SELECT on public.trends: the writer ran before default privileges "+ - "were set; repair with GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s, then re-run create", + return fmt.Errorf("%s cannot SELECT on public.trends: the table was created before the default "+ + "privileges were set; repair with GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s", ReaderRole, ReaderRole) } ok("%s holds SELECT on public.trends", ReaderRole) @@ -345,37 +371,5 @@ func (o Options) verify(ctx context.Context, asPartOfAll bool) error { return fmt.Errorf("%s holds INSERT on public.trends - the reader must be read-only", ReaderRole) } ok("%s does not hold INSERT", ReaderRole) - - if o.ReaderPassword != "" { - reader, err := o.connect(ctx, o.Database, ReaderRole, o.ReaderPassword) - if err != nil { - return fmt.Errorf("live reader connection: %w", err) - } - defer reader.Close(ctx) - var probed int - if err := reader.QueryRow(ctx, - "SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe").Scan(&probed); err != nil { - return fmt.Errorf("live reader probe: %w", err) - } - ok("live connection as %s reads public.trends", ReaderRole) - } else { - note("reader password not supplied - live connection as %s not tested.", ReaderRole) - } - - var defaultPartitionExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.tpdefault') IS NOT NULL").Scan(&defaultPartitionExists); err != nil { - return fmt.Errorf("checking public.tpdefault: %w", err) - } - if defaultPartitionExists { - var strayRows int64 - if err := archive.QueryRow(ctx, "SELECT count(*) FROM public.tpdefault").Scan(&strayRows); err != nil { - return fmt.Errorf("counting tpdefault rows: %w", err) - } - if strayRows > 0 { - warn("default partition tpdefault holds %d rows - a day partition was missing at write time.", strayRows) - } else { - ok("default partition tpdefault is empty") - } - } return nil } diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go index 9dfe0e4..d128acd 100644 --- a/internal/provision/provision_test.go +++ b/internal/provision/provision_test.go @@ -1,10 +1,13 @@ package provision import ( + "strings" "testing" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + + semibase "github.com/Semiteq/SemiBase" ) func TestValidate(t *testing.T) { @@ -243,3 +246,22 @@ func TestEndpointUsesTheConfiguredDatabase(t *testing.T) { t.Errorf("Endpoint() = %q, want %q", got, want) } } + +// the archive table is the vendor's shape, transcribed. This pins the parts a consumer +// depends on and the one object we deliberately do not create +func TestEmbeddedTrendsSQL(t *testing.T) { + required := []string{ + "CREATE TABLE public.trends", + "PARTITION BY RANGE (t)", + "ADD CONSTRAINT tpk PRIMARY KEY (id, l, t)", + "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT", + } + for _, fragment := range required { + if !strings.Contains(semibase.TrendsSQL, fragment) { + t.Errorf("trends.sql does not contain %q", fragment) + } + } + if strings.Contains(semibase.TrendsSQL, "public.messages") { + t.Error("trends.sql creates public.messages; nothing we ship reads it") + } +} diff --git a/sql/trends.sql b/sql/trends.sql new file mode 100644 index 0000000..9f94b28 --- /dev/null +++ b/sql/trends.sql @@ -0,0 +1,30 @@ +-- The Simple-Scada 2 archive table, transcribed from the vendor's own definition (a +-- customer archive dump read with `pg_restore --schema-only`). The shape is the vendor's, +-- not ours: this tool creates the table so the reader's access chain is complete before the +-- SCADA's first start, and never alters it afterwards. +-- +-- Applied while connected as `scada_writer`, so the table's owner is the role the SCADA +-- writes with and the reader's SELECT arrives through the default privileges set for that +-- role. A superuser-owned table would give the reader access for a different reason than a +-- site gets it. +-- +-- `messages` is not created: nothing we ship reads it, and every object we create is a +-- surface that can drift from the vendor. +-- +-- Day partitions `tpYYYYmMMdDD` are not created here — the SCADA creates them on a site, +-- the consumer's bench seeder creates them on a bench. Only the `tpdefault` catch-all is +-- created; rows landing in it signal a missing day partition. + +CREATE TABLE public.trends ( + id integer DEFAULT 0 NOT NULL, + l smallint DEFAULT 0 NOT NULL, + t timestamp(3) without time zone NOT NULL, + v double precision, + q integer NOT NULL +) +PARTITION BY RANGE (t); + +ALTER TABLE ONLY public.trends + ADD CONSTRAINT tpk PRIMARY KEY (id, l, t); + +CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT; From 73904acf9230cd9a67bd79e4c1c7b266a3c5ed4d Mon Sep 17 00:00:00 2001 From: Oleg <18466855+EvaTheSalmon@users.noreply.github.com> Date: Mon, 24 Aug 2026 13:44:54 +0300 Subject: [PATCH 2/2] fix(provision): prove the read, drop the writer login The tail check asked has_table_privilege, which reads one catalog bit. After REVOKE USAGE ON SCHEMA public FROM public it answers t while every read the reader issues fails with "permission denied for schema public" - measured on postgres:17-alpine. The read itself is the check now: SET ROLE semiplot_reader, then SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe. The catalog bit is asked only after that read has failed, to split a missing table grant from a schema the reader cannot enter; each answer prints its own repair, and the grant repair again names both statements and the re-run. Over TCP, with a reader password in the run, the reader also logs in and reads - the half SET ROLE cannot reach, being pg_hba admission and the password a consumer carries. It is skipped with a note on a socket host and without that password. public.trends is created under SET ROLE scada_writer instead of over a scada_writer login. The login made the run depend on pg_hba admitting that role: on a socket with "local all all peer", the default on Debian, Ubuntu and RHEL, the superuser phases succeeded and the run then died with SQLSTATE 28000, leaving the database, both roles, the grants and semiplot_tags behind. Both routes were measured to leave the same relowner and the same relacl, so nothing about the reader's access path changes. Creating the table therefore needs no writer password; that password is now needed only on a first run, to create the role, and the usage text says so. The re-run path no longer says "exists, left untouched" and stops there: it requires public.tpdefault, since without it a row no day partition covers is rejected, and reports the tpdefault row count, which on a months-old table is the only signal that a day partition was missing at write time. The pending-restart read moves to the last statement of the tail, where the docs already claimed it was, and both commands run it - bench tunes nothing, but it runs against servers a site run tuned and nobody restarted. The DDL test pins the whole statement text of sql/trends.sql, columns, types and defaults included, and its messages guard now matches the unqualified name. The provisioning-order step and the Russian commissioning runbook hedge the SCADA-meets-an-existing-trends assumption the way provisioning.md does. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 10 +- cmd/semibase/main.go | 8 +- docs/architecture/configuration.md | 6 +- docs/architecture/overview.md | 9 +- docs/architecture/provisioning.md | 60 ++++--- docs/deployment.md | 16 +- internal/provision/provision.go | 229 +++++++++++++++++++++------ internal/provision/provision_test.go | 47 ++++-- sql/trends.sql | 9 +- 9 files changed, 294 insertions(+), 100 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0709c58..5b4cb3b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,13 +107,15 @@ gofmt -w . # run before presenting changes; gofmt is authoritative `site` and `bench` differ in one thing: `site` applies the `ALTER SYSTEM` memory constants, `bench` does not. Both create the database, the roles, the grants, `semiplot_tags` and -`public.trends`, and both end by checking that `semiplot_reader` reads `public.trends` and -cannot write it. A failed check is a non-zero exit. +`public.trends`, and both end by actually reading `public.trends` as `semiplot_reader` and +checking that the same role holds no `INSERT`. A failed check is a non-zero exit. Passwords come from flags, env, or a `.env` file in the working directory (flag > env > `.env`; template `.env.example`): `SEMIBASE_SUPER_PASSWORD`, `SEMIBASE_WRITER_PASSWORD`, -`SEMIBASE_READER_PASSWORD`. The writer password is needed to create `public.trends`, which the -tool does over a `scada_writer` login so the table's owner is the role the SCADA writes with. +`SEMIBASE_READER_PASSWORD`. The writer and reader passwords are needed only on a first run, +which creates those roles; `public.trends` is created under `SET ROLE scada_writer` on the +superuser connection, so the table's owner is the role the SCADA writes with and no +`pg_hba.conf` line has to admit a `scada_writer` login. ## Layout diff --git a/cmd/semibase/main.go b/cmd/semibase/main.go index 7c2bd5d..e74665c 100644 --- a/cmd/semibase/main.go +++ b/cmd/semibase/main.go @@ -29,8 +29,8 @@ Commands: bench a throwaway container: the same, without the tuning version print the build revision -Both commands end by checking that semiplot_reader can read public.trends and -cannot write it; a failed check is a non-zero exit. +Both commands end by reading public.trends as semiplot_reader and checking that the +same role cannot write it; a failed check is a non-zero exit. Every step checks before it creates; re-running either command is safe. Passwords of existing roles change only when the corresponding flag or variable is set. @@ -41,6 +41,10 @@ directory (flag wins over environment, environment wins over .env): --writer-password SEMIBASE_WRITER_PASSWORD scada_writer --reader-password SEMIBASE_READER_PASSWORD semiplot_reader +A first run creates both roles and fails without their passwords. Once the roles exist, +the superuser password alone carries a run; the reader password then only decides +whether the run also tests the semiplot_reader login, which it does over TCP. + Run 'semibase --help' for the command's flags. ` diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index b997358..cec0670 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -14,8 +14,10 @@ configuration, and no setting depends on inspecting the host. survive a reinstall of `postgresql.conf`, and applies them with `pg_reload_conf()`. Every setting below except `shared_buffers` is reload-context or weaker and takes effect immediately; `shared_buffers` takes effect at the next service restart or reboot. The tuning phase prints the -server's own pending-restart list (`pg_settings.pending_restart`) and `site` warns about it again -on its last lines, so a missed restart surfaces during commissioning. +server's own pending-restart list (`pg_settings.pending_restart`), and the tail of the run reads it +again as its last statement, so a missed restart is the last thing the run says. `bench` reads it +too — it tunes nothing, but it runs against existing servers, and one that a `site` run tuned can +still be waiting for its restart. `semibase bench` writes none of them: the values are constants sized to the installation machine's hardware floor and mean nothing to a throwaway container. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 4eba938..95dd089 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -30,7 +30,7 @@ plaintext password in a client configuration file an acceptable risk. | --- | --- | | `cmd/semibase`, `internal/provision` | `semibase.exe` — idempotent provisioning in two commands, `site` and `bench`: instance tuning (`site` only), database, roles, grants, `semiplot_tags`, `public.trends`, and the reader-access checks that end every run | | `sql/semiplot_tags.sql` | DDL for the one object we invented, embedded into the binary | -| `sql/trends.sql` | The vendor's archive-table shape, transcribed and embedded; applied as `scada_writer` | +| `sql/trends.sql` | The vendor's archive-table shape, transcribed and embedded; applied under `SET ROLE scada_writer`, so the table's owner is the writer | | `Dockerfile` | `ghcr.io/semiteq/semibase` — the Linux binary alone on `scratch`, for consumers that layer it into a bench image | | `docs/architecture/` | The instance as it is: configuration deltas, provisioning order, ownership | @@ -49,8 +49,11 @@ The order matters, because the default privileges have to be in place before any `semiplot_tags` and `public.trends`; it ends by proving the reader reads `trends` and cannot write it, and warns while `shared_buffers` waits for a service restart. 3. Restart the PostgreSQL service or reboot the machine, so `shared_buffers` takes effect. -4. Point the Simple-Scada project at the database and start it once. It finds `trends` in place - and creates the day partitions and `messages` itself. +4. Point the Simple-Scada project at the database and start it once. It is expected to find + `trends` in place, write into it, and create the day partitions and `messages` itself — an + **unverified** assumption, with the experiment that settles it, in + `provisioning.md` ("Assumption: the SCADA meeting an existing `trends`"). Set + `log_statement = 'all'` for that first start and read back the DDL the SCADA issued. 5. Fill `semiplot_tags` with the variables to be trended. 6. Write the SemiPlot connection file, including the source time zone. diff --git a/docs/architecture/provisioning.md b/docs/architecture/provisioning.md index 0b9ddd3..3c18cd7 100644 --- a/docs/architecture/provisioning.md +++ b/docs/architecture/provisioning.md @@ -88,7 +88,7 @@ a test of what a site runs: | Archive database, both roles, grants, default privileges, `semiplot_tags` | yes | yes | | `public.trends` with the `tpdefault` partition, created as `scada_writer` | yes | yes | | Reader-access checks at the tail | yes | yes | -| Pending-restart warning at the tail | yes | no — nothing was tuned, so nothing can be pending | +| Pending-restart warning at the tail | yes | yes — `bench` tunes nothing, but it runs against existing servers too, and one a `site` run tuned can still be waiting for its restart | `version` (or `--version`) prints the build revision: the ldflags value (`-X main.revision=...`) when a release sets it, otherwise the VCS revision from @@ -149,9 +149,10 @@ The package that runs the SQL owns its guards; the CLI is a thin dispatcher. (`phaseTimeout`) — is owned by the CLI dispatcher in `cmd/semibase`. - **Pending-restart reporting.** The tool never touches the service manager. The tuning phase applies its deltas with `pg_reload_conf()` and reads `pg_settings.pending_restart` back from - the server; `site` reads it again at the tail, so the last thing an operator sees names a - pending `shared_buffers`. The operator restarts the service or reboots the machine — the one - manual step, and the tool says so instead of doing it. + the server; both commands read it again as the **last** statement of the tail, after the + access checks, so the last line either prints names a pending `shared_buffers`. The operator + restarts the service or reboots the machine — the one manual step, and the tool says so + instead of doing it. ## The reader-access chain @@ -166,9 +167,9 @@ ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer IN SCHEMA public This is in place **before** `public.trends` is created and before the writer first starts; tables `scada_writer` creates afterwards are readable automatically. If a table was created before the -statement ran, the repair is a one-time `GRANT SELECT ON ALL TABLES IN SCHEMA public` plus the -default-privileges statement for the partitions still to come — the tail check detects that state -and prints the repair. +statement ran, the repair is a one-time `GRANT SELECT ON ALL TABLES IN SCHEMA public` for the +tables that exist plus the default-privileges statement for the partitions still to come, then +another run of the tool — the tail check detects that state and prints all three. On PostgreSQL 15 and later the `public` schema no longer grants `CREATE` to everyone, so `create` grants it to `scada_writer` explicitly. On 14 the grant is redundant and harmless. @@ -179,11 +180,16 @@ On PostgreSQL 15 and later the `public` schema no longer grants `CREATE` to ever shape, transcribed from a customer archive dump. Three properties of how it is created carry the weight: -- **The role, more than the shape.** The table is created over a `scada_writer` login of its own, - never with `SET ROLE`, because the reader's `SELECT` has to arrive through the default privileges - set for that role a moment earlier. A superuser-owned table would give the reader access for a - different reason than a site gets it, and a bench would then test a different thing from - production. A run that finds no `public.trends` and no writer password fails and says so. +- **The role, more than the shape.** The table is created as `scada_writer`, because the reader's + `SELECT` has to arrive through the default privileges set for that role a moment earlier. A + superuser-owned table would give the reader access for a different reason than a site gets it, + and a bench would then test a different thing from production. The role is assumed with + `SET ROLE` on the superuser connection, not through a `scada_writer` login: measured on + `postgres:17-alpine`, both routes leave the same `relowner` and the same `relacl` + (`{scada_writer=arwdDxtm/scada_writer,semiplot_reader=r/scada_writer}`), while a login also has + to be admitted by `pg_hba.conf` — and `local all all peer`, the default on Debian, Ubuntu and + RHEL, refuses it on a unix socket. Creating the table therefore needs no writer password; the + writer password is needed only on a first run, to create the role itself. - **`messages` is not created.** Nothing we ship reads it, and every object we create is a surface that can drift from the vendor. - **Day partitions are not created.** `tpYYYYmMMdDD` belongs to the SCADA on a site and to the @@ -191,7 +197,12 @@ weight: no day partition exists lands somewhere instead of failing. Existence is checked first (`to_regclass('public.trends')`), so a second run leaves the table -untouched — including a table the SCADA has since altered. +untouched — including a table the SCADA has since altered. Untouched is not unread: on that path +the run reads back what it promises. `public.tpdefault` must be there, and its absence is a +failure with the one `CREATE TABLE ... PARTITION OF ... DEFAULT` that repairs it, because a table +without it rejects any row no day partition covers. Its row count is then reported — always zero +on the create path, but on a table that may be months old a non-empty `tpdefault` is the only +signal that a day partition was missing at write time, so it is a warning rather than a failure. ## Assumption: the SCADA meeting an existing `trends` @@ -209,11 +220,24 @@ replaced by what was observed and `sql/trends.sql` is reconsidered. Both commands end with them, and a failure is a non-zero exit — the run did not reach the state it promises. They are knowable at exit precisely because this tool creates `public.trends` itself: -1. `semiplot_reader` holds `SELECT` on `public.trends`. Failing this prints the one repair message - that turns a silent misconfiguration into an actionable one. -2. `semiplot_reader` does not hold `INSERT` on `public.trends` — the reader is read-only. -3. `site` only: no setting waits for a service restart (`pg_settings.pending_restart`) — a warning, - not a failure, so a pending `shared_buffers` never blocks the access checks. +1. **`semiplot_reader` reads `public.trends`.** The read itself, not a catalog bit: + `SET ROLE semiplot_reader` on the superuser connection, then + `SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe`. `has_table_privilege` + cannot carry this check — after `REVOKE USAGE ON SCHEMA public FROM public` it still answers + `t` while every read the reader issues fails with `permission denied for schema public` + (measured). The catalog bit is asked only when the read has already failed, to split a missing + table grant from a schema the reader cannot enter, and each answer prints its own repair. + `SET ROLE` rather than a login for the `pg_hba.conf` reason above. +2. **The `semiplot_reader` login reads it too**, over TCP, when the reader password is present in + the run. This is the half `SET ROLE` cannot reach: `pg_hba.conf` admitting the role and the + password a consumer will carry. It is skipped, with a printed note, when no reader password + was given, and on a socket host — `peer` is the platform default there and no consumer reads + over the socket. +3. `semiplot_reader` does not hold `INSERT` on `public.trends` — the reader is read-only. This one + stays a catalog question: a failed `INSERT` proves nothing about the next one. +4. No setting waits for a service restart (`pg_settings.pending_restart`) — a warning, not a + failure, so a pending `shared_buffers` never blocks the access checks. Both commands read it, + last, after the access checks. ## Development benches diff --git a/docs/deployment.md b/docs/deployment.md index 6c43821..41a211d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -63,7 +63,13 @@ SemiBase разворачивается одной утилитой — `semibas Объекты `semiplot_*` принадлежат `postgres`; заполнение `semiplot_tags` на пусконаладке выполняется от суперпользователя. Таблицу `public.trends` утилита создаёт от имени `scada_writer` — владельцем должна быть та же роль, от которой пишет SCADA, иначе читатель -получит доступ не тем путём, каким он работает в эксплуатации. +получит доступ не тем путём, каким он работает в эксплуатации. Роль берётся командой +`SET ROLE` на соединении суперпользователя, отдельный вход под `scada_writer` не выполняется: +владелец и права на таблицу получаются те же, а требований к `pg_hba.conf` не добавляется. + +Пароли ролей нужны только на первом запуске — том, который эти роли создаёт. Когда роли уже +есть, достаточно пароля суперпользователя; пароль читателя тогда лишь включает +дополнительную проверку входа под `semiplot_reader` (только по TCP). ## Порядок ввода в эксплуатацию @@ -93,8 +99,12 @@ winget install --id PostgreSQL.PostgreSQL.17 --exact `shared_buffers`, применяются сразу; `shared_buffers` — только после перезапуска. Утилита предупреждает об этом в последних строках вывода, пока перезапуск не сделан. -5. Запустить проект Simple-Scada против базы. Таблицу `trends` он застаёт готовой и дальше - создаёт дневные партиции и `messages` сам. +5. Запустить проект Simple-Scada против базы. Ожидается, что таблицу `trends` он застанет + готовой, оставит её как есть и дальше создаст дневные партиции и `messages` сам. **Это + предположение не проверено на реальном проекте.** На первом запуске включите + `log_statement = 'all'` и прочитайте в журнале PostgreSQL, какие DDL-команды SCADA + выдала: если она создаёт или изменяет `trends`, порядок ввода в эксплуатацию нужно + пересмотреть. 6. Заполнить `semiplot_tags` перечнем переменных для трендов. diff --git a/internal/provision/provision.go b/internal/provision/provision.go index dd84bd6..6bc4f6e 100644 --- a/internal/provision/provision.go +++ b/internal/provision/provision.go @@ -41,13 +41,6 @@ func (o Options) Validate() error { return nil } -// the tuning phase is the only thing that can leave a setting waiting for a service -// restart, so only the site run reads pending_restart back at the end -const ( - afterTuning = true - withoutTuning = false -) - // Site brings an installation machine to its commissioned state: memory tuning, then // everything the archive needs. func (o Options) Site(ctx context.Context) error { @@ -57,7 +50,7 @@ func (o Options) Site(ctx context.Context) error { if err := o.create(ctx); err != nil { return err } - return o.check(ctx, afterTuning) + return o.check(ctx) } // Bench brings a throwaway container to the same state minus the tuning: its memory @@ -66,7 +59,7 @@ func (o Options) Bench(ctx context.Context) error { if err := o.create(ctx); err != nil { return err } - return o.check(ctx, withoutTuning) + return o.check(ctx) } // pgx dials a unix socket exactly when pgconn.isAbsolutePath accepts the host. Widening this @@ -153,6 +146,11 @@ func alterRolePasswordStatement(name, password string) string { return fmt.Sprintf("ALTER ROLE %s PASSWORD '%s'", name, escapeLiteral(password)) } +func defaultPrivilegesStatement() string { + return fmt.Sprintf("ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA public GRANT SELECT ON TABLES TO %s", + WriterRole, ReaderRole) +} + func (o Options) create(ctx context.Context) error { if err := o.Validate(); err != nil { return err @@ -216,9 +214,7 @@ func (o Options) create(ctx context.Context) error { // the writer creates public.trends below and its partitions later, so SELECT on them // can only be granted ahead of time, through default privileges of their owner - defaultPrivileges := fmt.Sprintf( - "ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA public GRANT SELECT ON TABLES TO %s", - WriterRole, ReaderRole) + defaultPrivileges := defaultPrivilegesStatement() if _, err := archive.Exec(ctx, defaultPrivileges); err != nil { return fmt.Errorf("%s: %w", defaultPrivileges, err) } @@ -232,38 +228,91 @@ func (o Options) create(ctx context.Context) error { } ok("semiplot_tags in place, readable by %s", ReaderRole) - return o.ensureArchiveTable(ctx, archive) + return ensureArchiveTable(ctx, archive) } -// public.trends is created over a scada_writer login of its own, never with SET ROLE: the -// reader's SELECT has to arrive through the default privileges set for that role above, which -// is how a site gets it. A superuser-owned table would hand the reader access for a different -// reason, and a bench would then test something production does not do. -func (o Options) ensureArchiveTable(ctx context.Context, archive *pgx.Conn) error { +func relationExists(ctx context.Context, conn *pgx.Conn, name string) (bool, error) { var exists bool - if err := archive.QueryRow(ctx, - "SELECT to_regclass('public.trends') IS NOT NULL").Scan(&exists); err != nil { - return fmt.Errorf("checking public.trends: %w", err) + if err := conn.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists); err != nil { + return false, fmt.Errorf("checking %s: %w", name, err) + } + return exists, nil +} + +// runAs assumes role for the statements fn issues on conn, and drops back afterwards. The +// superuser connection is short-lived, but a leaked role would silently change what every +// later statement on it is allowed to do. +func runAs(ctx context.Context, conn *pgx.Conn, role string, fn func() error) error { + if _, err := conn.Exec(ctx, "SET ROLE "+role); err != nil { + return fmt.Errorf("assuming the %s role: %w", role, err) + } + err := fn() + if _, resetErr := conn.Exec(ctx, "RESET ROLE"); resetErr != nil && err == nil { + return fmt.Errorf("dropping the %s role: %w", role, resetErr) + } + return err +} + +func ensureArchiveTable(ctx context.Context, archive *pgx.Conn) error { + exists, err := relationExists(ctx, archive, "public.trends") + if err != nil { + return err } if exists { - ok("public.trends exists, left untouched") - return nil + return inspectArchiveTable(ctx, archive) } - if o.WriterPassword == "" { - return fmt.Errorf("public.trends does not exist and no %s password was given to create it "+ - "as that role; supply --writer-password or SEMIBASE_WRITER_PASSWORD", WriterRole) + return createArchiveTable(ctx, archive) +} + +// public.trends is created as scada_writer, so the table's owner is the role the SCADA writes +// with and the reader's SELECT arrives through the default privileges set for that role a +// moment earlier. A superuser-owned table would hand the reader access for a different reason, +// and a bench would then test something production does not do. +// +// The role is assumed with SET ROLE on the superuser connection rather than through a +// scada_writer login. Both produce the same owner and the same relacl - measured, not assumed - +// but a login also has to be admitted by pg_hba.conf, and "local all all peer", the default on +// Debian, Ubuntu and RHEL, refuses it on a unix socket. +func createArchiveTable(ctx context.Context, archive *pgx.Conn) error { + err := runAs(ctx, archive, WriterRole, func() error { + if _, err := archive.Exec(ctx, semibase.TrendsSQL); err != nil { + return fmt.Errorf("applying trends.sql as %s: %w", WriterRole, err) + } + return nil + }) + if err != nil { + return err } + ok("public.trends created as %s, partitioned by t, with the tpdefault partition", WriterRole) + return nil +} - writer, err := o.connect(ctx, o.Database, WriterRole, o.WriterPassword) +// an existing table is never altered - it may be one the SCADA has since changed - but the run +// still promises the state this tool describes, so what it promises is read back. +func inspectArchiveTable(ctx context.Context, archive *pgx.Conn) error { + ok("public.trends exists, left untouched") + + defaultPartition, err := relationExists(ctx, archive, "public.tpdefault") if err != nil { return err } - defer writer.Close(ctx) + if !defaultPartition { + return fmt.Errorf("public.trends exists without its default partition public.tpdefault: a row "+ + "whose timestamp no day partition covers is rejected at write time. Repair with "+ + "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT, issued as %s", WriterRole) + } - if _, err := writer.Exec(ctx, semibase.TrendsSQL); err != nil { - return fmt.Errorf("applying trends.sql as %s: %w", WriterRole, err) + // on the create path this is always zero; here the table can be months old, and a non-empty + // tpdefault is the only signal that a day partition was missing at write time + var strayRows int64 + if err := archive.QueryRow(ctx, "SELECT count(*) FROM public.tpdefault").Scan(&strayRows); err != nil { + return fmt.Errorf("counting public.tpdefault rows: %w", err) + } + if strayRows > 0 { + warn("default partition tpdefault holds %d rows - a day partition was missing at write time.", strayRows) + return nil } - ok("public.trends created by %s, partitioned by t, with the tpdefault partition", WriterRole) + ok("default partition tpdefault is present and empty") return nil } @@ -325,9 +374,13 @@ func (o Options) ensureDatabase(ctx context.Context, conn *pgx.Conn) error { return nil } +// the cheapest statement that still needs SELECT on public.trends and USAGE on the schema it +// lives in, whatever the table's size +const readerProbe = "SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe" + // the tail of every run. The tool created public.trends itself a moment ago, so the reader's // access to it is knowable at exit instead of after the SCADA's first start. -func (o Options) check(ctx context.Context, reportPendingRestart bool) error { +func (o Options) check(ctx context.Context) error { if err := o.Validate(); err != nil { return err } @@ -339,29 +392,63 @@ func (o Options) check(ctx context.Context, reportPendingRestart bool) error { } defer archive.Close(ctx) - if reportPendingRestart { - pending, err := pendingRestartSettings(ctx, archive) - if err != nil { - return err - } - if len(pending) > 0 { - warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", - strings.Join(pending, ", ")) - } + if err := assertReaderReads(ctx, archive); err != nil { + return err + } + if err := assertReaderCannotWrite(ctx, archive); err != nil { + return err + } + if err := o.checkReaderLogin(ctx); err != nil { + return err } + // last, so a pending shared_buffers is the last thing the run prints + return reportPendingRestart(ctx, archive) +} - var canSelect bool - if err := archive.QueryRow(ctx, - "SELECT has_table_privilege($1, 'public.trends', 'SELECT')", ReaderRole).Scan(&canSelect); err != nil { - return fmt.Errorf("checking reader SELECT privilege: %w", err) +// the load-bearing check: the read itself, issued as the reader. has_table_privilege answers +// from one catalog bit and stops there - it says yes for a reader that REVOKE USAGE ON SCHEMA +// public has locked out of the schema the table lives in, a state this tool has to fail on. +// SET ROLE rather than a login, for the pg_hba reason createArchiveTable states; the login is +// checked separately, where it can be. +func assertReaderReads(ctx context.Context, archive *pgx.Conn) error { + var probeErr error + if err := runAs(ctx, archive, ReaderRole, func() error { + var probed int + probeErr = archive.QueryRow(ctx, readerProbe).Scan(&probed) + return nil + }); err != nil { + return err } - if !canSelect { - return fmt.Errorf("%s cannot SELECT on public.trends: the table was created before the default "+ - "privileges were set; repair with GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s", - ReaderRole, ReaderRole) + if probeErr != nil { + return diagnoseFailedRead(ctx, archive, probeErr) } - ok("%s holds SELECT on public.trends", ReaderRole) + ok("%s reads public.trends", ReaderRole) + return nil +} + +// the catalog bit earns its place here and only here: it separates a missing table grant, whose +// repair is the default-privileges chain, from a schema the reader cannot enter +func diagnoseFailedRead(ctx context.Context, archive *pgx.Conn, probeErr error) error { + var granted bool + if err := archive.QueryRow(ctx, + "SELECT has_table_privilege($1, 'public.trends', 'SELECT')", ReaderRole).Scan(&granted); err != nil { + return fmt.Errorf("%s cannot read public.trends (%w) and its SELECT privilege could not be read: %w", + ReaderRole, probeErr, err) + } + if !granted { + return fmt.Errorf("%s cannot read public.trends: it holds no SELECT on the table, so the table was "+ + "created before the default privileges were set (%w). Repair with GRANT SELECT ON ALL TABLES IN "+ + "SCHEMA public TO %s for the tables that exist, then %s for the partitions still to come, then "+ + "re-run this command", ReaderRole, probeErr, ReaderRole, defaultPrivilegesStatement()) + } + return fmt.Errorf("%s holds SELECT on public.trends and still cannot read it: %w. The usual cause is a "+ + "revoked schema privilege; repair with GRANT USAGE ON SCHEMA public TO %s, then re-run this command", + ReaderRole, probeErr, ReaderRole) +} +// the reader is read-only. No probe can prove the absence of a privilege the way a read proves +// its presence, so this one stays a catalog question. +func assertReaderCannotWrite(ctx context.Context, archive *pgx.Conn) error { var canInsert bool if err := archive.QueryRow(ctx, "SELECT has_table_privilege($1, 'public.trends', 'INSERT')", ReaderRole).Scan(&canInsert); err != nil { @@ -373,3 +460,45 @@ func (o Options) check(ctx context.Context, reportPendingRestart bool) error { ok("%s does not hold INSERT", ReaderRole) return nil } + +// what SET ROLE cannot reach: pg_hba admitting the reader, and the password a consumer will +// carry. It needs that password, and it is skipped on a socket host - "local all all peer" is +// the platform default there, no consumer reads over the socket, and the socket path exists so +// this tool can run as an init script inside a postgres image. +func (o Options) checkReaderLogin(ctx context.Context) error { + switch { + case o.ReaderPassword == "": + note("no %s password given - the login was not tested, only the privileges behind it.", ReaderRole) + return nil + case isSocketHost(o.Host): + note("socket host - the %s login was not tested; a consumer connects over TCP.", ReaderRole) + return nil + } + + reader, err := o.connect(ctx, o.Database, ReaderRole, o.ReaderPassword) + if err != nil { + return err + } + defer reader.Close(ctx) + + var probed int + if err := reader.QueryRow(ctx, readerProbe).Scan(&probed); err != nil { + return fmt.Errorf("%s reads public.trends over its own login: %w", ReaderRole, err) + } + ok("%s logs in and reads public.trends", ReaderRole) + return nil +} + +// both commands report it: bench tunes nothing, but it can run against a server a previous +// site run tuned and nobody restarted +func reportPendingRestart(ctx context.Context, conn *pgx.Conn) error { + pending, err := pendingRestartSettings(ctx, conn) + if err != nil { + return err + } + if len(pending) > 0 { + warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", + strings.Join(pending, ", ")) + } + return nil +} diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go index d128acd..c1d29bf 100644 --- a/internal/provision/provision_test.go +++ b/internal/provision/provision_test.go @@ -247,21 +247,40 @@ func TestEndpointUsesTheConfiguredDatabase(t *testing.T) { } } -// the archive table is the vendor's shape, transcribed. This pins the parts a consumer -// depends on and the one object we deliberately do not create -func TestEmbeddedTrendsSQL(t *testing.T) { - required := []string{ - "CREATE TABLE public.trends", - "PARTITION BY RANGE (t)", - "ADD CONSTRAINT tpk PRIMARY KEY (id, l, t)", - "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT", - } - for _, fragment := range required { - if !strings.Contains(semibase.TrendsSQL, fragment) { - t.Errorf("trends.sql does not contain %q", fragment) +// the file's header prose names the objects the DDL deliberately leaves alone, so a claim +// about what the DDL creates has to be made against the statements only +func sqlStatementsOf(sql string) string { + kept := make([]string, 0, strings.Count(sql, "\n")) + for line := range strings.SplitSeq(sql, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue } + kept = append(kept, line) + } + return strings.Join(strings.Fields(strings.Join(kept, " ")), " ") +} + +// the archive table is the vendor's shape, transcribed by hand from a dump, and the whole +// statement text is pinned rather than a few fragments of it: a consumer that reads id, l, t, +// v and q gets wrong charts rather than an error when a type, a default or a column drifts. +// Reformatting the file is meant to fail this - re-read sql/semiplot_dev.sql in the consumer +// repository before changing the golden text. +func TestEmbeddedTrendsSQL(t *testing.T) { + want := "CREATE TABLE public.trends ( " + + "id integer DEFAULT 0 NOT NULL, " + + "l smallint DEFAULT 0 NOT NULL, " + + "t timestamp(3) without time zone NOT NULL, " + + "v double precision, " + + "q integer NOT NULL " + + ") PARTITION BY RANGE (t); " + + "ALTER TABLE ONLY public.trends ADD CONSTRAINT tpk PRIMARY KEY (id, l, t); " + + "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT;" + got := sqlStatementsOf(semibase.TrendsSQL) + if got != want { + t.Errorf("trends.sql statements =\n%q\nwant\n%q", got, want) } - if strings.Contains(semibase.TrendsSQL, "public.messages") { - t.Error("trends.sql creates public.messages; nothing we ship reads it") + // unqualified as well as public.messages: the guard is about the object, not the spelling + if strings.Contains(got, "messages") { + t.Error("trends.sql creates messages; nothing we ship reads it") } } diff --git a/sql/trends.sql b/sql/trends.sql index 9f94b28..93f97c0 100644 --- a/sql/trends.sql +++ b/sql/trends.sql @@ -3,10 +3,11 @@ -- not ours: this tool creates the table so the reader's access chain is complete before the -- SCADA's first start, and never alters it afterwards. -- --- Applied while connected as `scada_writer`, so the table's owner is the role the SCADA --- writes with and the reader's SELECT arrives through the default privileges set for that --- role. A superuser-owned table would give the reader access for a different reason than a --- site gets it. +-- Applied under `SET ROLE scada_writer`, so the table's owner is the role the SCADA writes +-- with and the reader's SELECT arrives through the default privileges set for that role. A +-- superuser-owned table would give the reader access for a different reason than a site gets +-- it. SET ROLE rather than a `scada_writer` login: the owner and the ACL come out identical, +-- and no pg_hba.conf line has to admit that role. -- -- `messages` is not created: nothing we ship reads it, and every object we create is a -- surface that can drift from the vendor.