diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d97bbd..8d23e61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,10 +55,10 @@ jobs: - name: Test run: go test -race ./... - # The bench path: the same binary provisions the service container the - # way it provisions production. Run twice to prove idempotency. "all" - # ends with verify, which reports "writer has not run" as a state, not - # a failure - the expected order before any SCADA has started. + # The site path, the only thing that covers the ALTER SYSTEM tuning: the + # same binary provisions the service container the way it provisions an + # installation machine. Run twice to prove idempotency - the second run + # finds every object in place and still exits on a passing access check. - name: Provision the service container env: SEMIBASE_SUPER_PASSWORD: ci-super @@ -66,15 +66,15 @@ jobs: SEMIBASE_READER_PASSWORD: ci-reader run: | go build -o semibase ./cmd/semibase - ./semibase all --database semiplot_dev --expected-major 17 - ./semibase all --database semiplot_dev --expected-major 17 + ./semibase site --database semiplot_dev --expected-major 17 + ./semibase site --database semiplot_dev --expected-major 17 # The socket path, which nothing else covers: the same binary, run the way # the official image runs an init script. During init the server has # listen_addresses empty and only /var/run/postgresql answers, superuser # auth is local trust, and the script's own "set -e" makes the entrypoint # abort on failure. So the container reaching a state where semiplot_reader - # selects over TCP proves create completed over the socket. + # selects over TCP proves bench completed over the socket. - name: Provision over a unix socket, as a postgres image init script env: SEMIBASE_WRITER_PASSWORD: ci-writer @@ -96,7 +96,7 @@ jobs: cat > initdb-semibase.sh <<'SCRIPT' #!/bin/sh set -e - /semibase create --host /var/run/postgresql --database semiplot_dev --expected-major 17 + /semibase bench --host /var/run/postgresql --database semiplot_dev --expected-major 17 SCRIPT # the entrypoint sources a non-executable init script instead of running it, # and a binary without the bit set cannot run at all diff --git a/CLAUDE.md b/CLAUDE.md index 81a253b..5b4cb3b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,10 +6,10 @@ provisions and configures the instance that hosts the Simple-Scada 2 archive: th CLI binary, `semibase.exe`, a container image carrying its Linux build, plus the instance's architecture docs. Deployment target: Windows. Language: Go (module `github.com/Semiteq/SemiBase`), driver -`pgx/v5`, `sql/semiplot_tags.sql` embedded via `go:embed`. Entry point: `cmd/semibase`. -The module is pure `pgx` and compiles for any GOOS; Linux builds serve the containerised -test bench of consumers (SemiPlot runs `create` against an ephemeral `postgres:17` container). -All commands run from the repository root. +`pgx/v5`, `sql/semiplot_tags.sql` and `sql/trends.sql` embedded via `go:embed`. Entry point: +`cmd/semibase`. The module is pure `pgx` and compiles for any GOOS; Linux builds serve the +containerised test bench of consumers (SemiPlot runs `bench` against an ephemeral `postgres:17` +container). All commands run from the repository root. ## Build @@ -80,13 +80,15 @@ golangci-lint run golangci-lint is pinned to 2.12.2 (`winget install GolangCI.golangci-lint --version 2.12.2`); config in `.golangci.yml`. CI (`.github/workflows/ci.yml`) runs `go build`, `go test -race`, and the same lint on `windows-latest` and `ubuntu-latest` for every push and pull request; -the Linux job also provisions a `postgres:17-alpine` service container by running `all` -twice — the bench path and the idempotency check in one step — and builds the container image, -so a broken `Dockerfile` fails on the pull request rather than at tag time. CI pushes nothing. +the Linux job also provisions a `postgres:17-alpine` service container by running `site` +twice — the tuning path and the idempotency check in one step — provisions a second container +with `bench` as a `postgres` image init script over the unix socket, and builds the container +image, so a broken `Dockerfile` fails on the pull request rather than at tag time. CI pushes +nothing. Unit tests live beside the source (`cmd/semibase/*_test.go`, `internal/provision/*_test.go`), -table-driven. There are no database-touching tests; the `verify` command is the integration -check, run against a live server. +table-driven. There are no database-touching tests in the repository; the integration check is +the reader-access check both commands run at the tail, against a live server. ## Format @@ -97,24 +99,31 @@ gofmt -w . # run before presenting changes; gofmt is authoritative ## Run ```powershell -.\semibase.exe --help # commands: config | create | verify | all | version -.\semibase.exe create --port 15432 --database semiplot_dev --expected-major 17 -.\semibase.exe verify --reader-password # post-writer proof of the reader access chain +.\semibase.exe --help # commands: site | bench | version +.\semibase.exe site # an installation machine: tuning, then everything else +.\semibase.exe bench --port 15432 --database semiplot_dev --expected-major 17 .\semibase.exe version # print the build revision ``` +`site` and `bench` differ in one thing: `site` applies the `ALTER SYSTEM` memory constants, +`bench` does not. Both create the database, the roles, the grants, `semiplot_tags` and +`public.trends`, and both end by actually reading `public.trends` as `semiplot_reader` and +checking that the same role holds no `INSERT`. A failed check is a non-zero exit. + Passwords come from flags, env, or a `.env` file in the working directory (flag > env > `.env`; template `.env.example`): `SEMIBASE_SUPER_PASSWORD`, `SEMIBASE_WRITER_PASSWORD`, -`SEMIBASE_READER_PASSWORD`. -`verify` failing with "writer has not run" before the SCADA's first start is the expected -order, not a bug. +`SEMIBASE_READER_PASSWORD`. The writer and reader passwords are needed only on a first run, +which creates those roles; `public.trends` is created under `SET ROLE scada_writer` on the +superuser connection, so the table's owner is the role the SCADA writes with and no +`pg_hba.conf` line has to admit a `scada_writer` login. ## Layout ``` cmd/semibase/ CLI entry point: command dispatch, flags, usage text -internal/provision/ the phases: config (ALTER SYSTEM), create (db/roles/grants), verify -sql/ DDL for objects we own (semiplot_tags), embedded into the binary +internal/provision/ the phases behind the two commands: config (ALTER SYSTEM), create + (db/roles/grants/semiplot_tags/trends), check (reader access) +sql/ embedded DDL: semiplot_tags (ours) and trends (the vendor's shape) docs/ human docs in Russian (enter at docs/readme.md) docs/architecture/ agent-facing design docs in English docs/plans/ dated implementation plans @@ -129,3 +138,6 @@ docs/plans/ dated implementation plans in `docs/plans/completed/`. - The archive schema itself belongs to the SCADA and is documented in the SemiPlot repository (`docs/architecture/scada-archive.md` there); this repository documents the instance around it. + `sql/trends.sql` is a transcription of the vendor's shape, created once and never altered — + whether the SCADA tolerates finding it already there is an unverified assumption, stated with + its experiment in `docs/architecture/provisioning.md`. diff --git a/README.md b/README.md index efbfa8e..073b8b9 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,6 @@ SemiBase — разворачиваемый PostgreSQL-сервис для ус # установка pgsql winget install --id PostgreSQL.PostgreSQL.17 --exact -# конфигурация сервера, архивная база, роли, права -.\semibase.exe all +# конфигурация сервера, архивная база, роли, права, таблица архива +.\semibase.exe site ``` diff --git a/cmd/semibase/main.go b/cmd/semibase/main.go index 47aeffe..e74665c 100644 --- a/cmd/semibase/main.go +++ b/cmd/semibase/main.go @@ -17,20 +17,22 @@ import ( "github.com/Semiteq/SemiBase/internal/provision" ) -const usage = `semibase - provisions the SemiBase PostgreSQL instance. +const usage = `semibase - brings a PostgreSQL instance to its provisioned state and exits. Usage: semibase [flags] Commands: - config apply the server configuration deltas (ALTER SYSTEM + reload); - shared_buffers takes effect at the next service restart - create create the archive database, the roles, the access chain, and semiplot_tags - verify post-writer checks: archive tables exist, the reader reads and cannot write - all config + create + verify + site an installation machine: apply the memory tuning (ALTER SYSTEM + reload), + then create the archive database, the roles, the access chain, + semiplot_tags and public.trends + bench a throwaway container: the same, without the tuning version print the build revision -Every step checks before it creates; re-running any command is safe. Passwords of +Both commands end by reading public.trends as semiplot_reader and checking that the +same role cannot write it; a failed check is a non-zero exit. + +Every step checks before it creates; re-running either command is safe. Passwords of existing roles change only when the corresponding flag or variable is set. Passwords come from flags, environment variables, or a .env file in the working @@ -39,6 +41,10 @@ directory (flag wins over environment, environment wins over .env): --writer-password SEMIBASE_WRITER_PASSWORD scada_writer --reader-password SEMIBASE_READER_PASSWORD semiplot_reader +A first run creates both roles and fails without their passwords. Once the roles exist, +the superuser password alone carries a run; the reader password then only decides +whether the run also tests the semiplot_reader login, which it does over TCP. + Run 'semibase --help' for the command's flags. ` @@ -55,10 +61,8 @@ var ( // only tests write to this map, to insert stubs var commands = map[string]func(context.Context, provision.Options) error{ - "config": func(ctx context.Context, options provision.Options) error { return options.Config(ctx) }, - "create": func(ctx context.Context, options provision.Options) error { return options.Create(ctx) }, - "verify": func(ctx context.Context, options provision.Options) error { return options.Verify(ctx) }, - "all": func(ctx context.Context, options provision.Options) error { return options.All(ctx) }, + "site": func(ctx context.Context, options provision.Options) error { return options.Site(ctx) }, + "bench": func(ctx context.Context, options provision.Options) error { return options.Bench(ctx) }, } func main() { @@ -141,8 +145,10 @@ func revisionFromSettings(settings []debug.BuildSetting) string { return vcsRevision } -// password flags default to empty so the usage output can only ever show the -// variable names, never their values +// One flag set serves both commands: site and bench differ only in the tuning phase, +// which reads no flag of its own, so every flag below is used by either command. +// Password flags default to empty so the usage output can only ever show the variable +// names, never their values. func newFlagSet(command string, options *provision.Options) *flag.FlagSet { flags := flag.NewFlagSet(command, flag.ContinueOnError) flags.SetOutput(errorOutput) diff --git a/cmd/semibase/main_test.go b/cmd/semibase/main_test.go index 3feee2e..8a36eb7 100644 --- a/cmd/semibase/main_test.go +++ b/cmd/semibase/main_test.go @@ -66,7 +66,7 @@ func TestFlagUsageHidesPasswordValues(t *testing.T) { } options := provision.Options{} - flags := newFlagSet("create", &options) + flags := newFlagSet("bench", &options) buffer := &bytes.Buffer{} flags.SetOutput(buffer) flags.PrintDefaults() @@ -134,7 +134,7 @@ func TestParseOptions(t *testing.T) { t.Setenv(name, value) } - options, err := parseOptions("create", tt.arguments) + options, err := parseOptions("bench", tt.arguments) switch { case tt.wantErr != nil: @@ -221,6 +221,22 @@ func TestRunVersionCommandExitsZero(t *testing.T) { } } +// the surface is two commands and nothing else: config, create, verify and all were +// removed with no alias and no tombstone +func TestCommandSurface(t *testing.T) { + want := map[string]bool{"site": true, "bench": true} + for name := range commands { + if !want[name] { + t.Errorf("unexpected command %q in the dispatch map", name) + } + } + for name := range want { + if _, known := commands[name]; !known { + t.Errorf("command %q is missing from the dispatch map", name) + } + } +} + func TestRunUnmappedCommandIsAnError(t *testing.T) { buffer := captureErrorOutput(t) @@ -247,8 +263,8 @@ func TestRunHelpFlagExitsZero(t *testing.T) { buffer := captureErrorOutput(t) clearPasswordEnvironment(t) - if got := run(context.Background(), []string{"create", "--help"}); got != 0 { - t.Fatalf("run(create --help) = %d, want exit code 0", got) + if got := run(context.Background(), []string{"bench", "--help"}); got != 0 { + t.Fatalf("run(bench --help) = %d, want exit code 0", got) } if !strings.Contains(buffer.String(), "-super-password") { t.Errorf("help output does not list the command flags: %q", buffer.String()) @@ -336,13 +352,13 @@ func TestCancelledContextReturnsPromptly(t *testing.T) { cancel() started := time.Now() - err := commands["create"](ctx, options) + err := commands["bench"](ctx, options) elapsed := time.Since(started) if !errors.Is(err, context.Canceled) { - t.Fatalf("create with cancelled context returned %v, want context.Canceled in the chain", err) + t.Fatalf("bench with cancelled context returned %v, want context.Canceled in the chain", err) } if elapsed > 2*time.Second { - t.Errorf("create with cancelled context took %v, want a prompt return", elapsed) + t.Errorf("bench with cancelled context took %v, want a prompt return", elapsed) } } diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 1ba3ceb..6905cef 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -8,8 +8,8 @@ Declarative architecture docs (English, present tense). These describe the syste - [overview.md](./overview.md) — purpose, consumers, components, provisioning order. - [configuration.md](./configuration.md) — every setting changed from the PostgreSQL default, with why. -- [provisioning.md](./provisioning.md) — the tool's commands, idempotency rules, and the - reader-access chain. +- [provisioning.md](./provisioning.md) — the tool's two commands, idempotency rules, the + reader-access chain, and the stated assumption about the SCADA meeting an existing `trends`. ## Locked decisions @@ -18,12 +18,13 @@ Declarative architecture docs (English, present tense). These describe the syste | Installation | PostgreSQL via `winget`, major version pinned, automatic upgrade disabled; engine install stays outside the tool | | Major version | 17 for new installs, pinned latest; floor 14 (SemiPlot needs `date_bin`; Simple-Scada 2 docs require 14+) | | Provisioning tool | Go single binary `semibase.exe` (`pgx/v5`, embedded SQL); no runtime or `psql` on the target machine | -| Instance configuration | Fixed `ALTER SYSTEM` deltas applied by `semibase config` via `pg_reload_conf()`, never by editing `postgresql.conf`; `shared_buffers` takes effect at the next service restart, reported through `pg_settings.pending_restart` | +| Instance configuration | Fixed `ALTER SYSTEM` deltas applied by `semibase site` via `pg_reload_conf()`, never by editing `postgresql.conf`; `shared_buffers` takes effect at the next service restart, reported through `pg_settings.pending_restart` | | Hardware floor | Installation machines guarantee 8 GB RAM and 50 GB database disk; memory settings are fixed constants sized to that floor, identical on every machine | | Roles | `scada_writer` (SCADA), `semiplot_reader` (viewers, `SELECT` only); `semiplot_*` objects owned by `postgres` — a dedicated owner role appears when a tag-editing mechanism exists | -| Reader access | `ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer` set **before** the writer first runs | -| Objects we add | `semiplot_tags` only — no triggers, functions, scheduled jobs, or extensions | -| Archive schema | Owned by Simple-Scada 2; documented in the SemiPlot repository, never created or altered here | +| Command surface | Two commands, `site` and `bench`, named for the situation rather than for the tool's internal steps; they differ in one thing, the memory tuning, which only `site` applies | +| Reader access | `ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer` set **before** `public.trends` is created, and before the writer first runs | +| Objects we add | `semiplot_tags` and the vendor-shaped `public.trends` — no triggers, functions, scheduled jobs, or extensions; `messages` is not created, nothing we ship reads it | +| Archive schema | Owned by Simple-Scada 2 and documented in the SemiPlot repository. This tool creates `public.trends` once, with the vendor's shape, connected as `scada_writer`, and never alters it afterwards; day partitions and every later change to the schema remain the vendor's | | Distribution | Binaries as GitHub release assets; the Linux binary also as `ghcr.io/semiteq/semibase`, `FROM scratch`, one file, pushed after the release exists; benches track `:latest` and a prerelease never moves it | | Backup method | `UNDECIDED` — commissioning-time, needs the customer's archive size | | Retention depth / disk sizing | `UNDECIDED` — needs a measured write rate from a working installation | diff --git a/docs/architecture/configuration.md b/docs/architecture/configuration.md index 3e8f0b7..cec0670 100644 --- a/docs/architecture/configuration.md +++ b/docs/architecture/configuration.md @@ -1,6 +1,6 @@ # Instance configuration -Every setting the `config` command writes is listed. Most differ from the PostgreSQL default; +Every setting the `site` command writes is listed. Most differ from the PostgreSQL default; two (`effective_cache_size`, `checkpoint_completion_target`) equal it and are pinned deliberately, so the values in force are visible here rather than implied. The workload is a sustained append-only insert stream from one writer plus a small number of read-heavy @@ -10,12 +10,17 @@ Every installation machine guarantees at least 8 GB of RAM and 50 GB of disk for so the memory figures are fixed constants sized to that floor: every machine runs the identical configuration, and no setting depends on inspecting the host. -The `config` command of `semibase.exe` writes these through `ALTER SYSTEM`, so they live in -`postgresql.auto.conf` and survive a reinstall of `postgresql.conf`, and applies them with -`pg_reload_conf()`. Every setting below except `shared_buffers` is reload-context or weaker and -takes effect immediately; `shared_buffers` takes effect at the next service restart or reboot. -`config` prints the server's own pending-restart list (`pg_settings.pending_restart`), and -`verify` warns while any setting still waits, so a missed restart surfaces during commissioning. +`semibase site` writes these through `ALTER SYSTEM`, so they live in `postgresql.auto.conf` and +survive a reinstall of `postgresql.conf`, and applies them with `pg_reload_conf()`. Every setting +below except `shared_buffers` is reload-context or weaker and takes effect immediately; +`shared_buffers` takes effect at the next service restart or reboot. The tuning phase prints the +server's own pending-restart list (`pg_settings.pending_restart`), and the tail of the run reads it +again as its last statement, so a missed restart is the last thing the run says. `bench` reads it +too — it tunes nothing, but it runs against existing servers, and one that a `site` run tuned can +still be waiting for its restart. + +`semibase bench` writes none of them: the values are constants sized to the installation +machine's hardware floor and mean nothing to a throwaway container. ## Server settings diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index f969860..95dd089 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -15,8 +15,8 @@ its own retention setting. | Consumer | Role | Access | | --- | --- | --- | -| Simple-Scada 2 (archives into PostgreSQL) | `scada_writer` | Creates and writes `trends`/`messages` and their partitions; executes retention | -| SemiPlot (trend viewer) | `semiplot_reader` | `SELECT` on `trends`, `messages`, `semiplot_tags` — nothing else | +| Simple-Scada 2 (archives into PostgreSQL) | `scada_writer` | Owns and writes `trends`, creates its day partitions, creates and writes `messages`; executes retention | +| SemiPlot (trend viewer) | `semiplot_reader` | `SELECT` on `trends` and `semiplot_tags` — nothing else | The `semiplot_*` objects are owned by `postgres`; commissioning fills `semiplot_tags` as the superuser. A dedicated owner role appears when a tag-editing mechanism exists to hold it. @@ -28,8 +28,9 @@ plaintext password in a client configuration file an acceptable risk. | Component | Responsibility | | --- | --- | -| `cmd/semibase`, `internal/provision` | `semibase.exe` — idempotent provisioning: instance configuration, database, roles, grants, `semiplot_tags`, post-writer verification | -| `sql/semiplot_tags.sql` | DDL for the one object we add to the archive database, embedded into the binary | +| `cmd/semibase`, `internal/provision` | `semibase.exe` — idempotent provisioning in two commands, `site` and `bench`: instance tuning (`site` only), database, roles, grants, `semiplot_tags`, `public.trends`, and the reader-access checks that end every run | +| `sql/semiplot_tags.sql` | DDL for the one object we invented, embedded into the binary | +| `sql/trends.sql` | The vendor's archive-table shape, transcribed and embedded; applied under `SET ROLE scada_writer`, so the table's owner is the writer | | `Dockerfile` | `ghcr.io/semiteq/semibase` — the Linux binary alone on `scratch`, for consumers that layer it into a bench image | | `docs/architecture/` | The instance as it is: configuration deltas, provisioning order, ownership | @@ -40,20 +41,27 @@ silently stop. ## Provisioning order -The order matters, because the archive tables do not exist until the SCADA has run once. +The order matters, because the default privileges have to be in place before any table +`scada_writer` owns is created. 1. Install the engine: `winget install --id PostgreSQL.PostgreSQL.17 --exact`. -2. `semibase config` — apply the configuration deltas; `shared_buffers` takes effect at the - next service restart or reboot, and `verify` warns while it waits. -3. `semibase create` — archive database, both roles, default privileges, `semiplot_tags`. -4. Point the Simple-Scada project at the database and start it once. It creates `trends`, - `messages` and the first daily partitions. -5. `semibase verify` — prove the reader access chain against the tables the writer created. -6. Fill `semiplot_tags` with the variables to be trended. -7. Write the SemiPlot connection file, including the source time zone. - -Every consumer must survive every intermediate state of this sequence — no database, database -without `trends`, `trends` without `semiplot_tags`, `semiplot_tags` without rows. Each is a normal +2. `semibase site` — tuning, then the archive database, both roles, default privileges, + `semiplot_tags` and `public.trends`; it ends by proving the reader reads `trends` and cannot + write it, and warns while `shared_buffers` waits for a service restart. +3. Restart the PostgreSQL service or reboot the machine, so `shared_buffers` takes effect. +4. Point the Simple-Scada project at the database and start it once. It is expected to find + `trends` in place, write into it, and create the day partitions and `messages` itself — an + **unverified** assumption, with the experiment that settles it, in + `provisioning.md` ("Assumption: the SCADA meeting an existing `trends`"). Set + `log_statement = 'all'` for that first start and read back the DDL the SCADA issued. +5. Fill `semiplot_tags` with the variables to be trended. +6. Write the SemiPlot connection file, including the source time zone. + +A bench replaces step 2 with `semibase bench` and has no step 3 or 4: the consumer's seeder plays +the writer's part. + +Every consumer must survive every intermediate state of this sequence — no database, empty +`trends`, `trends` without `semiplot_tags`, `semiplot_tags` without rows. Each is a normal condition, not a crash. ## Upgrades diff --git a/docs/architecture/provisioning.md b/docs/architecture/provisioning.md index 9e6556b..3c18cd7 100644 --- a/docs/architecture/provisioning.md +++ b/docs/architecture/provisioning.md @@ -7,8 +7,9 @@ matters, is a liability. The binary is self-contained: `pgx/v5` speaks the wire protocol itself, over TCP or a unix socket, so neither `psql.exe` nor any runtime needs to be present on the target machine. -`sql/semiplot_tags.sql` is embedded at build time. Installing the PostgreSQL engine itself is one documented `winget` -line, deliberately outside the tool — OS package management is not database provisioning. +`sql/semiplot_tags.sql` and `sql/trends.sql` are embedded at build time. Installing the PostgreSQL +engine itself is one documented `winget` line, deliberately outside the tool — OS package management +is not database provisioning. ## Distribution @@ -69,12 +70,25 @@ no say in it. ## Commands +The tool is a setup script: it brings an instance to a known state and exits. It is run once, at +commissioning, and nobody comes back to run a second command by hand. So the surface names the two +situations it is run in, not the phases it runs through. + | Command | What it does | When | | --- | --- | --- | -| `config` | Writes the `ALTER SYSTEM` deltas from `configuration.md` and applies them with `pg_reload_conf()`; prints the server's pending-restart list (`shared_buffers` is the one entry, active after the next service restart or reboot) | Production. Never against a shared dev server — it retunes the whole instance; a throwaway bench container needs no tuning | -| `create` | Creates the archive database, the roles, the grants, the default privileges, and `semiplot_tags` | Before the SCADA's first start | -| `verify` | Proves the reader access chain against the tables the writer created | After the SCADA's first start | -| `all` | `config` + `create` + `verify`; `verify` reports "writer has not run" as a distinct state, not a failure | Fresh instance | +| `site` | The memory tuning, then everything else | An installation machine | +| `bench` | Everything else, without the tuning | A throwaway container: a consumer's test bench or a development database | + +The two differ in exactly one thing, the tuning. The rest is identical, which is what makes a bench +a test of what a site runs: + +| Step | `site` | `bench` | +| --- | --- | --- | +| `ALTER SYSTEM` constants from `configuration.md`, applied with `pg_reload_conf()` | yes | no — the constants are sized for an installation machine and mean nothing to a container | +| Archive database, both roles, grants, default privileges, `semiplot_tags` | yes | yes | +| `public.trends` with the `tpdefault` partition, created as `scada_writer` | yes | yes | +| Reader-access checks at the tail | yes | yes | +| Pending-restart warning at the tail | yes | yes — `bench` tunes nothing, but it runs against existing servers too, and one a `site` run tuned can still be waiting for its restart | `version` (or `--version`) prints the build revision: the ldflags value (`-X main.revision=...`) when a release sets it, otherwise the VCS revision from @@ -104,7 +118,7 @@ pgx then dials `/.s.PGSQL.` on the `unix` network. Widening the would be a trap, not a courtesy: libpq also accepts a leading `@` for Linux's abstract namespace, but pgx resolves such a host as a TCP name, so `semibase` leaves it on the TCP path. -The socket form is what lets `create` run as an init script in the official `postgres` image, +The socket form is what lets `bench` run as an init script in the official `postgres` image, whose entrypoint serves `/docker-entrypoint-initdb.d/` from a temporary server with `listen_addresses` set to empty — reachable over the socket only. The Linux CI job runs exactly that: it layers the freshly built binary onto `postgres:17-alpine` behind such an init script and @@ -120,7 +134,7 @@ never exist. The package that runs the SQL owns its guards; the CLI is a thin dispatcher. - **Database-name validation.** `Options.Validate()` checks `Database` against - `^[a-z_][a-z0-9_]*$` and runs first in every phase (`Config`, `Create`, `verify`). The + `^[a-z_][a-z0-9_]*$` and runs first in every phase (`config`, `create`, `check`). The pattern lives only in `internal/provision`; the CLI calls the same method for the friendly early error. - **Identifier quoting.** Every interpolation of the database name into DDL goes through @@ -133,60 +147,116 @@ The package that runs the SQL owns its guards; the CLI is a thin dispatcher. in-flight query through pgx instead of killing the process mid-DDL. The context itself — `signal.NotifyContext(..., os.Interrupt)` plus the five-minute per-command timeout (`phaseTimeout`) — is owned by the CLI dispatcher in `cmd/semibase`. -- **Pending-restart reporting.** The tool never touches the service manager. `config` applies - its deltas with `pg_reload_conf()` and reads `pg_settings.pending_restart` back from the - server; `verify` warns while any setting still waits. The operator restarts the service or - reboots the machine — the one manual step, and the tool says so instead of doing it. +- **Pending-restart reporting.** The tool never touches the service manager. The tuning phase + applies its deltas with `pg_reload_conf()` and reads `pg_settings.pending_restart` back from + the server; both commands read it again as the **last** statement of the tail, after the + access checks, so the last line either prints names a pending `shared_buffers`. The operator + restarts the service or reboots the machine — the one manual step, and the tool says so + instead of doing it. ## The reader-access chain -`semiplot_reader` needs `SELECT` on `trends` and `messages` — tables that do not exist until the -SCADA has run once. A plain `GRANT` cannot be issued on a table that is not there, so `create` -sets: +`semiplot_reader` needs `SELECT` on `trends` and on every day partition the SCADA creates later — +objects that do not exist when the grant has to be decided. A plain `GRANT` cannot be issued on a +table that is not there, so the create phase sets: ```sql ALTER DEFAULT PRIVILEGES FOR ROLE scada_writer IN SCHEMA public GRANT SELECT ON TABLES TO semiplot_reader; ``` -This must be in place **before** the writer first starts; tables the writer creates afterwards are -readable automatically. If the writer ran first, the repair is a one-time -`GRANT SELECT ON ALL TABLES IN SCHEMA public` plus the default-privileges statement for the -partitions still to come — `verify` detects this state and says so. +This is in place **before** `public.trends` is created and before the writer first starts; tables +`scada_writer` creates afterwards are readable automatically. If a table was created before the +statement ran, the repair is a one-time `GRANT SELECT ON ALL TABLES IN SCHEMA public` for the +tables that exist plus the default-privileges statement for the partitions still to come, then +another run of the tool — the tail check detects that state and prints all three. On PostgreSQL 15 and later the `public` schema no longer grants `CREATE` to everyone, so `create` grants it to `scada_writer` explicitly. On 14 the grant is redundant and harmless. -## What `verify` proves - -1. `public.trends` and `public.messages` exist — the writer has run. -2. `semiplot_reader` holds `SELECT` and does not hold `INSERT` on `trends`. -3. When the reader password is supplied, an actual connection as `semiplot_reader` reads one row — - the whole chain, not just the catalog view of it. -4. The default partition is empty. Rows in it mean a day partition was missing at write time, - which is a writer-side fault worth surfacing during commissioning. -5. No setting waits for a service restart (`pg_settings.pending_restart`) — a warning, not a - failure, so a pending `shared_buffers` never blocks the access-chain checks. +## The archive table + +`public.trends` is created by this tool, by both commands, from `sql/trends.sql` — the vendor's +shape, transcribed from a customer archive dump. Three properties of how it is created carry the +weight: + +- **The role, more than the shape.** The table is created as `scada_writer`, because the reader's + `SELECT` has to arrive through the default privileges set for that role a moment earlier. A + superuser-owned table would give the reader access for a different reason than a site gets it, + and a bench would then test a different thing from production. The role is assumed with + `SET ROLE` on the superuser connection, not through a `scada_writer` login: measured on + `postgres:17-alpine`, both routes leave the same `relowner` and the same `relacl` + (`{scada_writer=arwdDxtm/scada_writer,semiplot_reader=r/scada_writer}`), while a login also has + to be admitted by `pg_hba.conf` — and `local all all peer`, the default on Debian, Ubuntu and + RHEL, refuses it on a unix socket. Creating the table therefore needs no writer password; the + writer password is needed only on a first run, to create the role itself. +- **`messages` is not created.** Nothing we ship reads it, and every object we create is a surface + that can drift from the vendor. +- **Day partitions are not created.** `tpYYYYmMMdDD` belongs to the SCADA on a site and to the + consumer's seeder on a bench. Only the `tpdefault` catch-all is created, so a row written while + no day partition exists lands somewhere instead of failing. + +Existence is checked first (`to_regclass('public.trends')`), so a second run leaves the table +untouched — including a table the SCADA has since altered. Untouched is not unread: on that path +the run reads back what it promises. `public.tpdefault` must be there, and its absence is a +failure with the one `CREATE TABLE ... PARTITION OF ... DEFAULT` that repairs it, because a table +without it rejects any row no day partition covers. Its row count is then reported — always zero +on the create path, but on a table that may be months old a non-empty `tpdefault` is the only +signal that a day partition was missing at write time, so it is a warning rather than a failure. + +## Assumption: the SCADA meeting an existing `trends` + +**Unverified.** Simple-Scada 2's behaviour when its archive target already exists is undocumented +and unmeasured. The expectation is that it writes into the existing table the way it does after a +reconnect, because the shape it finds is the shape it makes. + +The experiment that settles it: set `log_statement = 'all'`, start a Simple-Scada project once +against a database provisioned by `semibase site`, and read from the PostgreSQL log the DDL the +SCADA issues. If it creates the table unconditionally, or alters what it finds, this section is +replaced by what was observed and `sql/trends.sql` is reconsidered. + +## What the tail checks prove + +Both commands end with them, and a failure is a non-zero exit — the run did not reach the state it +promises. They are knowable at exit precisely because this tool creates `public.trends` itself: + +1. **`semiplot_reader` reads `public.trends`.** The read itself, not a catalog bit: + `SET ROLE semiplot_reader` on the superuser connection, then + `SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe`. `has_table_privilege` + cannot carry this check — after `REVOKE USAGE ON SCHEMA public FROM public` it still answers + `t` while every read the reader issues fails with `permission denied for schema public` + (measured). The catalog bit is asked only when the read has already failed, to split a missing + table grant from a schema the reader cannot enter, and each answer prints its own repair. + `SET ROLE` rather than a login for the `pg_hba.conf` reason above. +2. **The `semiplot_reader` login reads it too**, over TCP, when the reader password is present in + the run. This is the half `SET ROLE` cannot reach: `pg_hba.conf` admitting the role and the + password a consumer will carry. It is skipped, with a printed note, when no reader password + was given, and on a socket host — `peer` is the platform default there and no consumer reads + over the socket. +3. `semiplot_reader` does not hold `INSERT` on `public.trends` — the reader is read-only. This one + stays a catalog question: a failed `INSERT` proves nothing about the next one. +4. No setting waits for a service restart (`pg_settings.pending_restart`) — a warning, not a + failure, so a pending `shared_buffers` never blocks the access checks. Both commands read it, + last, after the access checks. ## Development benches The bench is an ephemeral vanilla `postgres:17-alpine` container. The consumer's test fixture -starts it and provisions it by running `create --database semiplot_dev --expected-major 17` — -the same code path production takes, which is what keeps the fixture from drifting into a -hand-written copy of the grants. `config` is not part of the bench: a throwaway container has -nothing to tune. 14 remains the floor `create` accepts through `--expected-major`; the bench -runs the pinned major, 17. +starts it and provisions it by running `bench --database semiplot_dev --expected-major 17` — the +same code path a site takes minus the tuning, which is what keeps the fixture from drifting into a +hand-written copy of the grants. 14 remains the floor accepted through `--expected-major`; the +bench runs the pinned major, 17. Two rules make the bench exercise what production exercises: -- The bench seeder connects **as `scada_writer`** and creates the archive tables itself, the way - the SCADA would. This is what makes the default-privileges chain a daily-tested path instead of - a commissioning-day surprise. +- `public.trends` is created **as `scada_writer`**, by both commands, so the default-privileges + chain is a daily-tested path instead of a commissioning-day surprise. - Bench integration tests read **as `semiplot_reader`**, never as a superuser, so a broken grant fails a test today. -The seeder and the tests live in the SemiPlot repository; this repository only promises them a -correctly provisioned database. +The consumer's seeder fills the archive and creates the day partitions its data needs; it and the +tests live in the SemiPlot repository. This repository promises them a provisioned database with an +empty `public.trends` in it. ## Passwords diff --git a/docs/deployment.md b/docs/deployment.md index 5570914..41a211d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,15 +1,15 @@ # Развёртывание -SemiBase разворачивается одной утилитой — `semibase.exe`. Утилита идемпотентна: каждый шаг -сначала проверяет состояние, затем создаёт недостающее. Повторный запуск любой команды -безопасен. Пароли существующих ролей меняются только при явной передаче соответствующего -флага или переменной окружения. +SemiBase разворачивается одной утилитой — `semibase.exe`. Это скрипт установки: одна команда +приводит экземпляр в заданное состояние и завершается. Утилита идемпотентна: каждый шаг сначала +проверяет состояние, затем создаёт недостающее. Повторный запуск безопасен. Пароли существующих +ролей меняются только при явной передаче соответствующего флага или переменной окружения. Справка: ```powershell .\semibase.exe --help # список команд -.\semibase.exe create --help # флаги конкретной команды +.\semibase.exe site --help # флаги конкретной команды ``` ## Требования @@ -29,13 +29,14 @@ SemiBase разворачивается одной утилитой — `semibas ## Команды -| Команда | Действие | -| --------- | --------------------------------------------------------------------------------------------------------------------------- | -| `config` | Настройки сервера через `ALTER SYSTEM` с немедленным применением; `shared_buffers` вступает в силу после перезапуска службы | -| `create` | Архивная база, роли, права доступа, таблица `semiplot_tags` | -| `verify` | Проверка после первого запуска SCADA: таблицы существуют, читатель читает и не пишет | -| `all` | `config` + `create` + `verify` | -| `version` | Печать ревизии сборки | +| Команда | Действие | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `site` | Машина установки: настройки сервера через `ALTER SYSTEM`, затем архивная база, роли, права доступа, таблицы `semiplot_tags` и `public.trends` | +| `bench` | Временный контейнер (тестовый стенд, база для разработки): то же самое без настроек сервера — константы памяти рассчитаны на машину установки и контейнеру ничего не дают | +| `version` | Печать ревизии сборки | + +Обе команды заканчиваются проверкой: `semiplot_reader` читает `public.trends` и не может в неё +писать. Ненулевой код возврата означает, что заявленное состояние не достигнуто. При обращении за поддержкой приложите вывод `semibase version` — он однозначно называет сборку утилиты на машине. @@ -56,16 +57,24 @@ SemiBase разворачивается одной утилитой — `semibas | Роль | Кто использует | Доступ | | ----------------- | -------------------------- | --------------------------------------------------- | -| `scada_writer` | Simple-Scada 2 | Создаёт и пишет архивные таблицы, выполняет ретеншн | -| `semiplot_reader` | SemiPlot и другие читатели | Только чтение `trends`, `messages`, `semiplot_tags` | +| `scada_writer` | Simple-Scada 2 | Владеет архивными таблицами и пишет в них, создаёт дневные партиции, выполняет ретеншн | +| `semiplot_reader` | SemiPlot и другие читатели | Только чтение `trends` и `semiplot_tags` | Объекты `semiplot_*` принадлежат `postgres`; заполнение `semiplot_tags` на пусконаладке -выполняется от суперпользователя. +выполняется от суперпользователя. Таблицу `public.trends` утилита создаёт от имени +`scada_writer` — владельцем должна быть та же роль, от которой пишет SCADA, иначе читатель +получит доступ не тем путём, каким он работает в эксплуатации. Роль берётся командой +`SET ROLE` на соединении суперпользователя, отдельный вход под `scada_writer` не выполняется: +владелец и права на таблицу получаются те же, а требований к `pg_hba.conf` не добавляется. + +Пароли ролей нужны только на первом запуске — том, который эти роли создаёт. Когда роли уже +есть, достаточно пароля суперпользователя; пароль читателя тогда лишь включает +дополнительную проверку входа под `semiplot_reader` (только по TCP). ## Порядок ввода в эксплуатацию -Порядок важен: архивные таблицы появляются только после первого запуска SCADA, а права -на чтение для них выставляются заранее, командой `create`. +Порядок важен: права на чтение выставляются до создания `public.trends`, поэтому утилита +отрабатывает до первого запуска SCADA. 1. Установить СУБД: @@ -75,32 +84,35 @@ winget install --id PostgreSQL.PostgreSQL.17 --exact 2. Задать пароли ролей: скопировать `.env.example` в `.env` рядом с утилитой и заполнить. -3. Настроить сервер и создать базу, роли и права: +3. Выполнить одну команду: ```powershell -.\semibase.exe config -.\semibase.exe create +.\semibase.exe site ``` -Все настройки, кроме `shared_buffers`, применяются сразу; `shared_buffers` — после -перезапуска службы PostgreSQL или перезагрузки машины. Перезагрузка обязательна до сдачи; -`verify` предупреждает, пока она не сделана. - -4. Запустить проект Simple-Scada против базы один раз — он создаёт `trends`, `messages` - и первые дневные партиции. +На выходе утилита гарантирует: настройки сервера применены, архивная база, обе роли, права +и права по умолчанию созданы, `semiplot_tags` и `public.trends` (с партицией `tpdefault`) +на месте, `semiplot_reader` читает `public.trends` и не может в неё писать. Отдельной +проверки после этого не требуется — она уже выполнена, её результат в коде возврата. -5. Проверить цепочку доступа читателя: +4. Перезапустить службу PostgreSQL или перезагрузить машину. Все настройки, кроме + `shared_buffers`, применяются сразу; `shared_buffers` — только после перезапуска. + Утилита предупреждает об этом в последних строках вывода, пока перезапуск не сделан. -```powershell -.\semibase.exe verify -``` +5. Запустить проект Simple-Scada против базы. Ожидается, что таблицу `trends` он застанет + готовой, оставит её как есть и дальше создаст дневные партиции и `messages` сам. **Это + предположение не проверено на реальном проекте.** На первом запуске включите + `log_statement = 'all'` и прочитайте в журнале PostgreSQL, какие DDL-команды SCADA + выдала: если она создаёт или изменяет `trends`, порядок ввода в эксплуатацию нужно + пересмотреть. 6. Заполнить `semiplot_tags` перечнем переменных для трендов. 7. Записать файл подключения SemiPlot, включая часовой пояс источника. -Если `verify` сообщает, что читатель не имеет доступа к `trends`, значит SCADA была -запущена раньше команды `create`. Способ починки утилита называет в тексте ошибки. +Если проверка в конце сообщает, что `semiplot_reader` не имеет доступа к `trends`, значит +таблица была создана раньше, чем выставлены права по умолчанию. Способ починки утилита +называет в тексте ошибки. ## Основные флаги diff --git a/docs/readme.md b/docs/readme.md index 60bf81b..5d1b1b5 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,4 +1,4 @@ # Документация SemiBase -- [Развёртывание](./deployment.md) — требования, установка СУБД, фазы скрипта, параметры, +- [Развёртывание](./deployment.md) — требования, установка СУБД, команды утилиты, параметры, порядок ввода в эксплуатацию. diff --git a/embed.go b/embed.go index 4def537..08ed30e 100644 --- a/embed.go +++ b/embed.go @@ -4,3 +4,6 @@ import _ "embed" //go:embed sql/semiplot_tags.sql var SemiplotTagsSQL string + +//go:embed sql/trends.sql +var TrendsSQL string diff --git a/internal/provision/config.go b/internal/provision/config.go index 21026d9..cdc2ec0 100644 --- a/internal/provision/config.go +++ b/internal/provision/config.go @@ -27,7 +27,7 @@ var settings = []setting{ {"track_io_timing", "on"}, } -func (o Options) Config(ctx context.Context) error { +func (o Options) config(ctx context.Context) error { if err := o.Validate(); err != nil { return err } diff --git a/internal/provision/provision.go b/internal/provision/provision.go index ff7b509..6bc4f6e 100644 --- a/internal/provision/provision.go +++ b/internal/provision/provision.go @@ -41,18 +41,25 @@ func (o Options) Validate() error { return nil } -func (o Options) All(ctx context.Context) error { - if err := o.Config(ctx); err != nil { +// Site brings an installation machine to its commissioned state: memory tuning, then +// everything the archive needs. +func (o Options) Site(ctx context.Context) error { + if err := o.config(ctx); err != nil { return err } - if err := o.Create(ctx); err != nil { + if err := o.create(ctx); err != nil { return err } - return o.verify(ctx, true) + return o.check(ctx) } -func (o Options) Verify(ctx context.Context) error { - return o.verify(ctx, false) +// Bench brings a throwaway container to the same state minus the tuning: its memory +// constants are sized for an installation machine and mean nothing to a container. +func (o Options) Bench(ctx context.Context) error { + if err := o.create(ctx); err != nil { + return err + } + return o.check(ctx) } // pgx dials a unix socket exactly when pgconn.isAbsolutePath accepts the host. Widening this @@ -139,11 +146,16 @@ func alterRolePasswordStatement(name, password string) string { return fmt.Sprintf("ALTER ROLE %s PASSWORD '%s'", name, escapeLiteral(password)) } -func (o Options) Create(ctx context.Context) error { +func defaultPrivilegesStatement() string { + return fmt.Sprintf("ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA public GRANT SELECT ON TABLES TO %s", + WriterRole, ReaderRole) +} + +func (o Options) create(ctx context.Context) error { if err := o.Validate(); err != nil { return err } - step("Phase create: database %s, roles, access chain", o.Database) + step("Phase create: database %s, roles, access chain, archive table", o.Database) super, err := o.connect(ctx, "postgres", o.SuperUser, o.SuperPassword) if err != nil { @@ -200,11 +212,9 @@ func (o Options) Create(ctx context.Context) error { return fmt.Errorf("granting CREATE on schema public: %w", err) } - // trends/messages do not exist until the SCADA runs, so SELECT on them can only be - // granted ahead of time, through default privileges of their future owner - defaultPrivileges := fmt.Sprintf( - "ALTER DEFAULT PRIVILEGES FOR ROLE %s IN SCHEMA public GRANT SELECT ON TABLES TO %s", - WriterRole, ReaderRole) + // the writer creates public.trends below and its partitions later, so SELECT on them + // can only be granted ahead of time, through default privileges of their owner + defaultPrivileges := defaultPrivilegesStatement() if _, err := archive.Exec(ctx, defaultPrivileges); err != nil { return fmt.Errorf("%s: %w", defaultPrivileges, err) } @@ -217,6 +227,92 @@ func (o Options) Create(ctx context.Context) error { return fmt.Errorf("granting SELECT on semiplot_tags: %w", err) } ok("semiplot_tags in place, readable by %s", ReaderRole) + + return ensureArchiveTable(ctx, archive) +} + +func relationExists(ctx context.Context, conn *pgx.Conn, name string) (bool, error) { + var exists bool + if err := conn.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", name).Scan(&exists); err != nil { + return false, fmt.Errorf("checking %s: %w", name, err) + } + return exists, nil +} + +// runAs assumes role for the statements fn issues on conn, and drops back afterwards. The +// superuser connection is short-lived, but a leaked role would silently change what every +// later statement on it is allowed to do. +func runAs(ctx context.Context, conn *pgx.Conn, role string, fn func() error) error { + if _, err := conn.Exec(ctx, "SET ROLE "+role); err != nil { + return fmt.Errorf("assuming the %s role: %w", role, err) + } + err := fn() + if _, resetErr := conn.Exec(ctx, "RESET ROLE"); resetErr != nil && err == nil { + return fmt.Errorf("dropping the %s role: %w", role, resetErr) + } + return err +} + +func ensureArchiveTable(ctx context.Context, archive *pgx.Conn) error { + exists, err := relationExists(ctx, archive, "public.trends") + if err != nil { + return err + } + if exists { + return inspectArchiveTable(ctx, archive) + } + return createArchiveTable(ctx, archive) +} + +// public.trends is created as scada_writer, so the table's owner is the role the SCADA writes +// with and the reader's SELECT arrives through the default privileges set for that role a +// moment earlier. A superuser-owned table would hand the reader access for a different reason, +// and a bench would then test something production does not do. +// +// The role is assumed with SET ROLE on the superuser connection rather than through a +// scada_writer login. Both produce the same owner and the same relacl - measured, not assumed - +// but a login also has to be admitted by pg_hba.conf, and "local all all peer", the default on +// Debian, Ubuntu and RHEL, refuses it on a unix socket. +func createArchiveTable(ctx context.Context, archive *pgx.Conn) error { + err := runAs(ctx, archive, WriterRole, func() error { + if _, err := archive.Exec(ctx, semibase.TrendsSQL); err != nil { + return fmt.Errorf("applying trends.sql as %s: %w", WriterRole, err) + } + return nil + }) + if err != nil { + return err + } + ok("public.trends created as %s, partitioned by t, with the tpdefault partition", WriterRole) + return nil +} + +// an existing table is never altered - it may be one the SCADA has since changed - but the run +// still promises the state this tool describes, so what it promises is read back. +func inspectArchiveTable(ctx context.Context, archive *pgx.Conn) error { + ok("public.trends exists, left untouched") + + defaultPartition, err := relationExists(ctx, archive, "public.tpdefault") + if err != nil { + return err + } + if !defaultPartition { + return fmt.Errorf("public.trends exists without its default partition public.tpdefault: a row "+ + "whose timestamp no day partition covers is rejected at write time. Repair with "+ + "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT, issued as %s", WriterRole) + } + + // on the create path this is always zero; here the table can be months old, and a non-empty + // tpdefault is the only signal that a day partition was missing at write time + var strayRows int64 + if err := archive.QueryRow(ctx, "SELECT count(*) FROM public.tpdefault").Scan(&strayRows); err != nil { + return fmt.Errorf("counting public.tpdefault rows: %w", err) + } + if strayRows > 0 { + warn("default partition tpdefault holds %d rows - a day partition was missing at write time.", strayRows) + return nil + } + ok("default partition tpdefault is present and empty") return nil } @@ -278,11 +374,17 @@ func (o Options) ensureDatabase(ctx context.Context, conn *pgx.Conn) error { return nil } -func (o Options) verify(ctx context.Context, asPartOfAll bool) error { +// the cheapest statement that still needs SELECT on public.trends and USAGE on the schema it +// lives in, whatever the table's size +const readerProbe = "SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe" + +// the tail of every run. The tool created public.trends itself a moment ago, so the reader's +// access to it is knowable at exit instead of after the SCADA's first start. +func (o Options) check(ctx context.Context) error { if err := o.Validate(); err != nil { return err } - step("Phase verify: post-writer checks") + step("Phase check: the reader access chain") archive, err := o.connect(ctx, o.Database, o.SuperUser, o.SuperPassword) if err != nil { @@ -290,52 +392,63 @@ func (o Options) verify(ctx context.Context, asPartOfAll bool) error { } defer archive.Close(ctx) - pending, err := pendingRestartSettings(ctx, archive) - if err != nil { + if err := assertReaderReads(ctx, archive); err != nil { return err } - if len(pending) > 0 { - warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", - strings.Join(pending, ", ")) - } - - var trendsExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.trends') IS NOT NULL").Scan(&trendsExists); err != nil { - return fmt.Errorf("checking public.trends: %w", err) + if err := assertReaderCannotWrite(ctx, archive); err != nil { + return err } - if !trendsExists { - message := "writer has not run yet: public.trends does not exist. " + - "Start the Simple-Scada project against this database once, then run verify." - if asPartOfAll { - note("%s", message) - return nil - } - return fmt.Errorf("%s", message) + if err := o.checkReaderLogin(ctx); err != nil { + return err } - ok("public.trends exists") + // last, so a pending shared_buffers is the last thing the run prints + return reportPendingRestart(ctx, archive) +} - var messagesExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.messages') IS NOT NULL").Scan(&messagesExists); err != nil { - return fmt.Errorf("checking public.messages: %w", err) +// the load-bearing check: the read itself, issued as the reader. has_table_privilege answers +// from one catalog bit and stops there - it says yes for a reader that REVOKE USAGE ON SCHEMA +// public has locked out of the schema the table lives in, a state this tool has to fail on. +// SET ROLE rather than a login, for the pg_hba reason createArchiveTable states; the login is +// checked separately, where it can be. +func assertReaderReads(ctx context.Context, archive *pgx.Conn) error { + var probeErr error + if err := runAs(ctx, archive, ReaderRole, func() error { + var probed int + probeErr = archive.QueryRow(ctx, readerProbe).Scan(&probed) + return nil + }); err != nil { + return err } - if messagesExists { - ok("public.messages exists") - } else { - warn("public.messages does not exist") + if probeErr != nil { + return diagnoseFailedRead(ctx, archive, probeErr) } + ok("%s reads public.trends", ReaderRole) + return nil +} - var canSelect bool +// the catalog bit earns its place here and only here: it separates a missing table grant, whose +// repair is the default-privileges chain, from a schema the reader cannot enter +func diagnoseFailedRead(ctx context.Context, archive *pgx.Conn, probeErr error) error { + var granted bool if err := archive.QueryRow(ctx, - "SELECT has_table_privilege($1, 'public.trends', 'SELECT')", ReaderRole).Scan(&canSelect); err != nil { - return fmt.Errorf("checking reader SELECT privilege: %w", err) - } - if !canSelect { - return fmt.Errorf("%s cannot SELECT on public.trends: the writer ran before default privileges "+ - "were set; repair with GRANT SELECT ON ALL TABLES IN SCHEMA public TO %s, then re-run create", - ReaderRole, ReaderRole) - } - ok("%s holds SELECT on public.trends", ReaderRole) + "SELECT has_table_privilege($1, 'public.trends', 'SELECT')", ReaderRole).Scan(&granted); err != nil { + return fmt.Errorf("%s cannot read public.trends (%w) and its SELECT privilege could not be read: %w", + ReaderRole, probeErr, err) + } + if !granted { + return fmt.Errorf("%s cannot read public.trends: it holds no SELECT on the table, so the table was "+ + "created before the default privileges were set (%w). Repair with GRANT SELECT ON ALL TABLES IN "+ + "SCHEMA public TO %s for the tables that exist, then %s for the partitions still to come, then "+ + "re-run this command", ReaderRole, probeErr, ReaderRole, defaultPrivilegesStatement()) + } + return fmt.Errorf("%s holds SELECT on public.trends and still cannot read it: %w. The usual cause is a "+ + "revoked schema privilege; repair with GRANT USAGE ON SCHEMA public TO %s, then re-run this command", + ReaderRole, probeErr, ReaderRole) +} +// the reader is read-only. No probe can prove the absence of a privilege the way a read proves +// its presence, so this one stays a catalog question. +func assertReaderCannotWrite(ctx context.Context, archive *pgx.Conn) error { var canInsert bool if err := archive.QueryRow(ctx, "SELECT has_table_privilege($1, 'public.trends', 'INSERT')", ReaderRole).Scan(&canInsert); err != nil { @@ -345,37 +458,47 @@ func (o Options) verify(ctx context.Context, asPartOfAll bool) error { return fmt.Errorf("%s holds INSERT on public.trends - the reader must be read-only", ReaderRole) } ok("%s does not hold INSERT", ReaderRole) + return nil +} - if o.ReaderPassword != "" { - reader, err := o.connect(ctx, o.Database, ReaderRole, o.ReaderPassword) - if err != nil { - return fmt.Errorf("live reader connection: %w", err) - } - defer reader.Close(ctx) - var probed int - if err := reader.QueryRow(ctx, - "SELECT count(*) FROM (SELECT 1 FROM public.trends LIMIT 1) probe").Scan(&probed); err != nil { - return fmt.Errorf("live reader probe: %w", err) - } - ok("live connection as %s reads public.trends", ReaderRole) - } else { - note("reader password not supplied - live connection as %s not tested.", ReaderRole) +// what SET ROLE cannot reach: pg_hba admitting the reader, and the password a consumer will +// carry. It needs that password, and it is skipped on a socket host - "local all all peer" is +// the platform default there, no consumer reads over the socket, and the socket path exists so +// this tool can run as an init script inside a postgres image. +func (o Options) checkReaderLogin(ctx context.Context) error { + switch { + case o.ReaderPassword == "": + note("no %s password given - the login was not tested, only the privileges behind it.", ReaderRole) + return nil + case isSocketHost(o.Host): + note("socket host - the %s login was not tested; a consumer connects over TCP.", ReaderRole) + return nil } - var defaultPartitionExists bool - if err := archive.QueryRow(ctx, "SELECT to_regclass('public.tpdefault') IS NOT NULL").Scan(&defaultPartitionExists); err != nil { - return fmt.Errorf("checking public.tpdefault: %w", err) + reader, err := o.connect(ctx, o.Database, ReaderRole, o.ReaderPassword) + if err != nil { + return err } - if defaultPartitionExists { - var strayRows int64 - if err := archive.QueryRow(ctx, "SELECT count(*) FROM public.tpdefault").Scan(&strayRows); err != nil { - return fmt.Errorf("counting tpdefault rows: %w", err) - } - if strayRows > 0 { - warn("default partition tpdefault holds %d rows - a day partition was missing at write time.", strayRows) - } else { - ok("default partition tpdefault is empty") - } + defer reader.Close(ctx) + + var probed int + if err := reader.QueryRow(ctx, readerProbe).Scan(&probed); err != nil { + return fmt.Errorf("%s reads public.trends over its own login: %w", ReaderRole, err) + } + ok("%s logs in and reads public.trends", ReaderRole) + return nil +} + +// both commands report it: bench tunes nothing, but it can run against a server a previous +// site run tuned and nobody restarted +func reportPendingRestart(ctx context.Context, conn *pgx.Conn) error { + pending, err := pendingRestartSettings(ctx, conn) + if err != nil { + return err + } + if len(pending) > 0 { + warn("settings waiting for a service restart: %s - restart the PostgreSQL service or reboot the machine.", + strings.Join(pending, ", ")) } return nil } diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go index 9dfe0e4..c1d29bf 100644 --- a/internal/provision/provision_test.go +++ b/internal/provision/provision_test.go @@ -1,10 +1,13 @@ package provision import ( + "strings" "testing" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + + semibase "github.com/Semiteq/SemiBase" ) func TestValidate(t *testing.T) { @@ -243,3 +246,41 @@ func TestEndpointUsesTheConfiguredDatabase(t *testing.T) { t.Errorf("Endpoint() = %q, want %q", got, want) } } + +// the file's header prose names the objects the DDL deliberately leaves alone, so a claim +// about what the DDL creates has to be made against the statements only +func sqlStatementsOf(sql string) string { + kept := make([]string, 0, strings.Count(sql, "\n")) + for line := range strings.SplitSeq(sql, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + kept = append(kept, line) + } + return strings.Join(strings.Fields(strings.Join(kept, " ")), " ") +} + +// the archive table is the vendor's shape, transcribed by hand from a dump, and the whole +// statement text is pinned rather than a few fragments of it: a consumer that reads id, l, t, +// v and q gets wrong charts rather than an error when a type, a default or a column drifts. +// Reformatting the file is meant to fail this - re-read sql/semiplot_dev.sql in the consumer +// repository before changing the golden text. +func TestEmbeddedTrendsSQL(t *testing.T) { + want := "CREATE TABLE public.trends ( " + + "id integer DEFAULT 0 NOT NULL, " + + "l smallint DEFAULT 0 NOT NULL, " + + "t timestamp(3) without time zone NOT NULL, " + + "v double precision, " + + "q integer NOT NULL " + + ") PARTITION BY RANGE (t); " + + "ALTER TABLE ONLY public.trends ADD CONSTRAINT tpk PRIMARY KEY (id, l, t); " + + "CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT;" + got := sqlStatementsOf(semibase.TrendsSQL) + if got != want { + t.Errorf("trends.sql statements =\n%q\nwant\n%q", got, want) + } + // unqualified as well as public.messages: the guard is about the object, not the spelling + if strings.Contains(got, "messages") { + t.Error("trends.sql creates messages; nothing we ship reads it") + } +} diff --git a/sql/trends.sql b/sql/trends.sql new file mode 100644 index 0000000..93f97c0 --- /dev/null +++ b/sql/trends.sql @@ -0,0 +1,31 @@ +-- The Simple-Scada 2 archive table, transcribed from the vendor's own definition (a +-- customer archive dump read with `pg_restore --schema-only`). The shape is the vendor's, +-- not ours: this tool creates the table so the reader's access chain is complete before the +-- SCADA's first start, and never alters it afterwards. +-- +-- Applied under `SET ROLE scada_writer`, so the table's owner is the role the SCADA writes +-- with and the reader's SELECT arrives through the default privileges set for that role. A +-- superuser-owned table would give the reader access for a different reason than a site gets +-- it. SET ROLE rather than a `scada_writer` login: the owner and the ACL come out identical, +-- and no pg_hba.conf line has to admit that role. +-- +-- `messages` is not created: nothing we ship reads it, and every object we create is a +-- surface that can drift from the vendor. +-- +-- Day partitions `tpYYYYmMMdDD` are not created here — the SCADA creates them on a site, +-- the consumer's bench seeder creates them on a bench. Only the `tpdefault` catch-all is +-- created; rows landing in it signal a missing day partition. + +CREATE TABLE public.trends ( + id integer DEFAULT 0 NOT NULL, + l smallint DEFAULT 0 NOT NULL, + t timestamp(3) without time zone NOT NULL, + v double precision, + q integer NOT NULL +) +PARTITION BY RANGE (t); + +ALTER TABLE ONLY public.trends + ADD CONSTRAINT tpk PRIMARY KEY (id, l, t); + +CREATE TABLE public.tpdefault PARTITION OF public.trends DEFAULT;