From 98a1ed9271a9a4b537e9d4be3d2babe5de74e255 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 13:57:55 +0200 Subject: [PATCH 1/4] Expose account allowance and owner admission status Add a CLI view of exact non-monetary allowance totals and expose owner-scoped admission with explicitly unknown host drain status. Document quote revalidation and the operator-issued allowance policy. GitHub issues #83, #214 and #215. --- api/openapi.yaml | 6 ++ cmd/dbl/allowance.go | 41 +++++++++++ cmd/dbl/allowance_test.go | 72 +++++++++++++++++++ cmd/dbl/cli.go | 1 + cmd/dbl/commands.go | 2 + docs/api.md | 22 ++++++ docs/cli.md | 18 ++++- docs/operations/configuration.md | 9 +++ .../transport/api/handlers_operator.go | 29 +++++--- .../transport/api/handlers_operator_test.go | 30 +++++++- 10 files changed, 216 insertions(+), 14 deletions(-) create mode 100644 cmd/dbl/allowance.go create mode 100644 cmd/dbl/allowance_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index bf5d94d8..d6daf9ec 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -6031,6 +6031,12 @@ components: description: "pending means no certificate is bound; offline means enrolled without a ready control session; online means ready. Clients must tolerate future values." ready: type: "boolean" + admission: + type: "string" + description: "ready, maintenance (dispatcher admission paused) or offline (no ready control session, including pending enrollment). A snapshot, not a capacity guarantee; absent on older dispatchers." + drain_status: + type: "string" + description: "unknown: the control protocol does not observe the executor host's joined service shutdown. Offline and dispatcher maintenance never prove a safe executor drain. Absent on older dispatchers; clients must treat absent or unrecognized values as unavailable." last_seen: type: "integer" format: "int64" diff --git a/cmd/dbl/allowance.go b/cmd/dbl/allowance.go new file mode 100644 index 00000000..b11bede5 --- /dev/null +++ b/cmd/dbl/allowance.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "context" + "fmt" + "io" +) + +const allowanceUsage = `Usage: + dbl [--dispatcher NAME] allowance + +Report the account's granted, reserved, consumed and remaining TEST units. +These are non-transferable usage credits, not money. Requires a saved account +credential and a dispatcher offering allowances (API 1.15 or newer). +` + +func allowanceCommand(ctx context.Context, args []string, options globalOptions, stdout, stderr io.Writer) int { + fs := newCommandFlagSet("allowance") + if code, ok := parseCommandFlags(fs, args, allowanceUsage, stdout, stderr); !ok { + return code + } + if fs.NArg() != 0 { + return usageError("dbl allowance", allowanceUsage, stderr, "allowance takes no arguments") + } + c, code, ok := connect(ctx, "dbl allowance", options, false, stderr) + if !ok { + return code + } + allowance, err := c.Allowance(ctx) + if err != nil { + return reportFailure(ctx, "dbl allowance", stderr, err) + } + return emitReported(ctx, "dbl allowance", options.Output, stdout, stderr, allowance, func(w io.Writer) error { + _, err := fmt.Fprintf(w, "currency: %s (non-monetary usage units)\ngranted: %s\nreserved: %s\nconsumed: %s\nremaining: %s\npricing rule: %s\n", + allowance.Currency, allowance.Granted, allowance.Reserved, allowance.Consumed, allowance.Remaining, allowance.PricingRule) + return err + }) +} diff --git a/cmd/dbl/allowance_test.go b/cmd/dbl/allowance_test.go new file mode 100644 index 00000000..6191d134 --- /dev/null +++ b/cmd/dbl/allowance_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "context" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/netsec-ethz/debuglet/internal/connections" + "github.com/netsec-ethz/debuglet/pkg/client" +) + +func TestAllowanceUsesSelectedAccountAndExactUnits(t *testing.T) { + const token = "allowance-private-credential" + want := client.Allowance{Currency: "TEST", Granted: "9007199254740993", Reserved: "4", Consumed: "9007199254740991", Remaining: "-2", PricingRule: "bw-s-ceil-ms-v1"} + for _, mode := range []string{outputHuman, outputJSON} { + t.Run(mode, func(t *testing.T) { + fx := newFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/me/allowance" || r.Header.Get("Authorization") != "Bearer "+token || r.Header.Get("Debuglet-API-Version") != "1.15" { + t.Errorf("wrong allowance request: %s %s", r.Method, r.URL.Path) + } + writeJSONResponse(w, http.StatusOK, want) + })) + path := filepath.Join(t.TempDir(), "connections.json") + if err := connections.Save(path, connections.Profile{Name: "selected", Endpoint: fx.endpoint()}, true); err != nil { + t.Fatal(err) + } + if err := connections.SaveCredential(path, "selected", connections.Credential{Endpoint: fx.endpoint(), Token: token}); err != nil { + t.Fatal(err) + } + code, out, errout := runCLI(context.Background(), "--config", path, "--dispatcher", "selected", "--output", mode, "allowance") + assertCode(t, code, exitOK, out, errout) + if strings.Contains(out+errout, token) || errout != "" || fx.total() != 1 { + t.Fatalf("unexpected allowance output or requests: %q / %q / %d", out, errout, fx.total()) + } + if mode == outputJSON { + got := oneJSONDocument(t, out) + if got["granted"] != want.Granted || got["remaining"] != want.Remaining || got["currency"] != "TEST" { + t.Fatalf("allowance values lost: %s", out) + } + } else if !strings.Contains(out, "granted: "+want.Granted) || !strings.Contains(out, "remaining: -2") || !strings.Contains(out, "non-monetary") { + t.Fatalf("allowance values or units missing: %s", out) + } + }) + } +} + +func TestAllowanceFailureDoesNotPrintABalance(t *testing.T) { + for _, tc := range []struct { + name string + status int + body any + message string + }{ + {"disabled", http.StatusNotFound, map[string]string{"code": "not_found", "message": "allowances are not enabled"}, "allowances are not enabled"}, + {"unauthenticated", http.StatusUnauthorized, map[string]string{"code": "unauthorized", "message": "authentication required"}, "dbl login"}, + {"incomplete", http.StatusOK, client.Allowance{Currency: "TEST"}, "incomplete allowance"}, + } { + t.Run(tc.name, func(t *testing.T) { + fx := newFixture(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writeJSONResponse(w, tc.status, tc.body) })) + code, out, errout := runCLI(context.Background(), "--endpoint", fx.endpoint(), "allowance") + assertCode(t, code, exitFailure, out, errout) + if out != "" || !strings.Contains(errout, tc.message) { + t.Fatalf("invented balance or missing diagnostic: %q / %q", out, errout) + } + }) + } +} diff --git a/cmd/dbl/cli.go b/cmd/dbl/cli.go index 77ddf6cf..88208d5d 100644 --- a/cmd/dbl/cli.go +++ b/cmd/dbl/cli.go @@ -54,6 +54,7 @@ Commands: obtain and store a session credential logout revoke and forget the stored credential whoami show the selected session's account + allowance show the account's remaining TEST units config --role ROLE --file FILE inspect redacted daemon configuration doctor [--role ROLE --file FILE] [--connection] diagnose local setup or a saved connection diff --git a/cmd/dbl/commands.go b/cmd/dbl/commands.go index 47034c25..ada49067 100644 --- a/cmd/dbl/commands.go +++ b/cmd/dbl/commands.go @@ -76,6 +76,8 @@ func dispatch(ctx context.Context, command string, args []string, options global return logoutCommand(ctx, args, options, stdout, stderr) case "whoami": return whoamiCommand(ctx, args, options, stdout, stderr) + case "allowance": + return allowanceCommand(ctx, args, options, stdout, stderr) case "config": return configCommand(ctx, args, options, stdout, stderr) case "doctor": diff --git a/docs/api.md b/docs/api.md index 08f8b129..e95a01c7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -42,6 +42,17 @@ Creating a `USDC` intent requires `refund_address` to be a Sui address: `0x` fol The intent prices the batch again when it is created and returns the result as `quote` in its response, in the same shape, so a client can compare it with an earlier quote; a changed `pricing_rule` means the server's rule changed in between. An intent recovered through an explicit retry does not repeat its quote. +A quote is an immediate snapshot, with no promised validity interval, expiry +timestamp or price lock. Editing the batch invalidates the client's displayed +review; fetch a fresh quote for the exact edited batch. Before payment or run +submission, compare the intent's rule, currency, units, total and per-order +prices with that review. If they differ, show the new price and require a new +review; do not pay or submit automatically. If an intent expires before use, +discard it and request a fresh quote and intent. Quote generation and intent +creation both recheck supported policy and price; normal submission still +checks current capacity. A successful quote does not reserve capacity or +guarantee admission. There is no server-side quote token to expire or consume. + `GET /me` adds `economics`: `payment_methods` lists the methods the intent admits now (`TEST`, and `USDC` when blockchain payments are enabled), `chain_payments` whether blockchain payments are enabled and `allowances` whether usage allowances are enabled on this dispatcher (see [usage allowances](#usage-allowances-api-115)). `GET /me/orders` pages the caller's own payment intents, newest first. An intent has no stored creation time, so the order is that of its expiry time, which is creation plus five minutes, then of its ID. `limit` is 1 to 100 (default 25); pass the `next` value of a page as `before` to read the following one, and `next` is empty on the last page. Each intent reports `id`, `method`, `status` (`outstanding`, `paid`, `refunded`, `expired`, `aborted`, or `unknown`), its recorded `price` and `currency`, `pricing_rule` (empty when the dispatcher did not record one) and `expires_at`, and each of its orders `order_id`, `executor_id`, `price`, `currency`, `settlement` (`pending`, `credited` or `refunded`, as in run detail costs) and, once admitted, `run_id`. Another account's intents are never listed; a `before` value that is not one of your intents returns an empty page. @@ -134,6 +145,17 @@ API 1.10 adds authenticated `GET` and `POST /operator/executors`, plus operations scoped to the caller's machines; they do not grant dispatcher-wide operator privileges. Inventory includes pending and offline machines. +Each inventory entry separately reports `admission`: `ready` (subject to normal +admission checks), `maintenance` (the dispatcher has paused new submissions), or +`offline` (no ready control session, including pending enrollment). `ready` and +`status` continue to describe the control connection even during dispatcher +maintenance. `drain_status` is `unknown`: the control protocol does not observe +the executor host's service manager or completed joined shutdown. Display this +as unavailable, including when the field is absent or unrecognized; neither +offline nor maintenance proves an executor was safely drained. Confirm a drain +on the executor host with the [managed service procedure](operations/services.md). +These read-only fields do not grant remote drain or service-control authority. + `POST /executor-enrollment` exchanges a single-use setup token and a signed CSR for a machine certificate. It uses the token in the request body rather than a user session. The executor creates and retains its own private key. See diff --git a/docs/cli.md b/docs/cli.md index 2ef0381c..4ad3809c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -170,6 +170,22 @@ dbl --dispatcher research --output json whoami expired or revoked credential exits 1 with a `dbl login` hint. Credentials are never printed. +### Inspect the account's usage allowance + +```sh +dbl --dispatcher research allowance +dbl --dispatcher research --output json allowance +``` + +`allowance` reads the account's granted, reserved, consumed and remaining TEST +units through API 1.15 or newer. These are non-transferable usage credits, not +money or USDC. Amounts remain exact integer strings in JSON; a negative remaining +amount from earlier usage is reported as recorded. Reading creates no grant, +reservation or run. Disabled allowances or a missing/revoked credential exit 1 +with the dispatcher's diagnostic; an authentication failure includes a login hint. +Grants are issued by an operator and never renew automatically. See +[usage allowances](api.md#usage-allowances-api-115) for the accounting rules. + ### Inspect daemon configuration ```sh @@ -271,7 +287,7 @@ See [probe verification](verification.md). | --- | --- | | Run local roles | `demo`, `up`, `dispatcher up`, `executor up` | | Manage connections | `connect`, `dispatcher list`, `dispatcher use`, `dispatcher remove` | -| Manage credentials | `login`, `logout`, `whoami` | +| Manage credentials and inspect usage | `login`, `logout`, `whoami`, `allowance` | | Submit work | `validate`, `run`, `retry`, `rendezvous`, `cancel` | | Read results | `nodes`, `status`, `logs`, `recovery` | | Verify received probes | `verify` | diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md index 8f18fb24..214611df 100644 --- a/docs/operations/configuration.md +++ b/docs/operations/configuration.md @@ -35,6 +35,15 @@ The optional `[allowance]` section caps what authenticated accounts may reserve Nothing is granted automatically, at sign-up or later. An operator account grants an account an amount with a reason and an idempotency key through `POST /operator/accounts/{id}/allowance`; a request without an amount grants `default_grant`, which at a price of 1 covers ten runs of 100,000 bit/s for 30 seconds. Grants are never changed, reset or renewed, and an account's ceiling is the sum of its grants. While allowances are enabled, an account's `TEST` intent is created only if its remaining allowance covers the price, otherwise it is refused with `allowance_exceeded`; the request without a credential that the local development profile admits is not capped. See [the API reference](../api.md#usage-allowances-api-115) for how reservations are counted. +The operator decides who receives a grant and records its reason. Authentication +does not establish one account per person: allowances provide no Sybil resistance +on their own. Review related accounts and prior grants before issuing additional +units; a different idempotency key intentionally creates another grant. There is +no automatic refill, hosting reward or cash redemption. Users can inspect the +recorded totals with `dbl allowance` or the SDK's `Allowance` method. Accounts +without grants cannot reserve positive-cost work when allowances are enabled; +zero-cost work still remains subject to the ordinary admission quotas. + ### Dispatcher blockchain payments The `[sui]` section configures chain payments. Every shipped configuration sets `disabled = true` and leaves the other fields empty; in that mode they are ignored and the dispatcher serves `TEST` payments only. `server.local_development = true` requires `disabled = true`. Enabling chain payments is an operator decision with its own profile and checklist; see [chain payments](payments.md). diff --git a/internal/dispatcher/transport/api/handlers_operator.go b/internal/dispatcher/transport/api/handlers_operator.go index a79a3f23..f4da2938 100644 --- a/internal/dispatcher/transport/api/handlers_operator.go +++ b/internal/dispatcher/transport/api/handlers_operator.go @@ -14,18 +14,23 @@ import ( "github.com/google/uuid" "github.com/labstack/echo/v4" + "github.com/netsec-ethz/debuglet/internal/dispatcher" "github.com/netsec-ethz/debuglet/internal/dispatcher/database" "github.com/netsec-ethz/debuglet/internal/dispatcher/enrollment" "github.com/netsec-ethz/debuglet/pkg/wire" ) type OwnedExecutorResponse struct { - ID string `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - Ready bool `json:"ready"` - LastSeen int64 `json:"last_seen,omitempty"` - Version string `json:"version,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Ready bool `json:"ready"` + Admission string `json:"admission"` + // DrainStatus is unknown: the control protocol does not report whether + // the executor's host service manager completed a joined shutdown. + DrainStatus string `json:"drain_status"` + LastSeen int64 `json:"last_seen,omitempty"` + Version string `json:"version,omitempty"` } type OwnedExecutorsResponse struct { @@ -43,12 +48,13 @@ type ExecutorSetupResponse struct { func (h *Handler) onboardingEnabled() bool { return h.onboarding.Enabled && h.issuer != nil } -func (h *Handler) ownedExecutor(id, name string, enrolled bool) OwnedExecutorResponse { - result := OwnedExecutorResponse{ID: id, Name: name, Status: "pending"} +func (h *Handler) ownedExecutor(id, name string, enrolled, paused bool) OwnedExecutorResponse { + result := OwnedExecutorResponse{ID: id, Name: name, Status: "pending", Admission: wire.AdmissionOffline, DrainStatus: "unknown"} if enrolled { result.Status = "offline" if node, ok := h.dispatcher.GetExecutor(id); ok { result.Ready, result.Version = node.Ready, node.Version + result.Admission = node.Admission(paused) if !node.LastSeen.IsZero() { result.LastSeen = node.LastSeen.Unix() } @@ -73,8 +79,9 @@ func (h *Handler) GetOwnedExecutors(c echo.Context) error { if response.Enabled { response.DispatcherURL = h.onboarding.DispatcherURL } + paused := dispatcher.AdmissionPaused() != nil for _, row := range rows { - response.Executors = append(response.Executors, h.ownedExecutor(row.ExecutorID, row.Name, row.Enrolled)) + response.Executors = append(response.Executors, h.ownedExecutor(row.ExecutorID, row.Name, row.Enrolled, paused)) } return c.JSON(http.StatusOK, response) } @@ -111,7 +118,7 @@ func (h *Handler) PostOwnedExecutor(c echo.Context) error { } c.Response().Header().Set("Cache-Control", "no-store") return c.JSON(http.StatusCreated, ExecutorSetupResponse{ - Executor: h.ownedExecutor(id, request.Name, false), Token: token, + Executor: h.ownedExecutor(id, request.Name, false, dispatcher.AdmissionPaused() != nil), Token: token, ExpiresAt: expires.Unix(), DispatcherURL: h.onboarding.DispatcherURL, }) } @@ -138,7 +145,7 @@ func (h *Handler) PostOwnedExecutorToken(c echo.Context) error { } c.Response().Header().Set("Cache-Control", "no-store") return c.JSON(http.StatusOK, ExecutorSetupResponse{ - Executor: h.ownedExecutor(row.ExecutorID, row.Name, row.Enrolled), Token: token, + Executor: h.ownedExecutor(row.ExecutorID, row.Name, row.Enrolled, dispatcher.AdmissionPaused() != nil), Token: token, ExpiresAt: expires.Unix(), DispatcherURL: h.onboarding.DispatcherURL, }) } diff --git a/internal/dispatcher/transport/api/handlers_operator_test.go b/internal/dispatcher/transport/api/handlers_operator_test.go index 3047a22d..d6c1afd3 100644 --- a/internal/dispatcher/transport/api/handlers_operator_test.go +++ b/internal/dispatcher/transport/api/handlers_operator_test.go @@ -22,9 +22,12 @@ import ( "time" "github.com/google/uuid" + "github.com/netsec-ethz/debuglet/internal/demo/service" + "github.com/netsec-ethz/debuglet/internal/dispatcher" "github.com/netsec-ethz/debuglet/internal/dispatcher/config" "github.com/netsec-ethz/debuglet/internal/dispatcher/database" "github.com/netsec-ethz/debuglet/internal/dispatcher/enrollment" + "github.com/netsec-ethz/debuglet/pkg/wire" pb "github.com/netsec-ethz/debuglet/protocol" ) @@ -95,6 +98,8 @@ func operatorRequest(t *testing.T, f *ccFixture, token, method, path string, pay func TestOwnedExecutorEnrollmentAndRecovery(t *testing.T) { f, csr := operatorFixture(t) + switchFile := filepath.Join(t.TempDir(), "maintenance") + t.Setenv(dispatcher.MaintenanceFileEnv, switchFile) _, alice, _ := authAccount(t, f, "Alice") _, bob, _ := authAccount(t, f, "Bob") var setup ExecutorSetupResponse @@ -106,6 +111,9 @@ func TestOwnedExecutorEnrollmentAndRecovery(t *testing.T) { if setup.Executor.Name != "Alice's node" || setup.Executor.Status != "pending" || setup.Executor.Ready || setup.Token == "" || setup.ExpiresAt < time.Now().Add(23*time.Hour).Unix() { t.Fatalf("unexpected setup: %+v", setup.Executor) } + if setup.Executor.Admission != wire.AdmissionOffline || setup.Executor.DrainStatus != "unknown" { + t.Fatalf("pending executor admission or drain invented: %+v", setup.Executor) + } var list OwnedExecutorsResponse operatorRequest(t, f, bob, http.MethodGet, "/operator/executors", nil, 200, &list) if !list.Enabled || len(list.Executors) != 0 { @@ -138,7 +146,7 @@ func TestOwnedExecutorEnrollmentAndRecovery(t *testing.T) { } operatorRequest(t, f, "", http.MethodPost, "/executor-enrollment", request, 401, nil) operatorRequest(t, f, alice, http.MethodGet, "/operator/executors", nil, 200, &list) - if list.Executors[0].Status != "offline" || list.Executors[0].Ready { + if list.Executors[0].Status != "offline" || list.Executors[0].Ready || list.Executors[0].Admission != wire.AdmissionOffline || list.Executors[0].DrainStatus != "unknown" { t.Fatalf("issued certificate was reported connected: %+v", list) } @@ -168,9 +176,27 @@ func TestOwnedExecutorEnrollmentAndRecovery(t *testing.T) { t.Fatal(err) } operatorRequest(t, f, alice, http.MethodGet, "/operator/executors", nil, 200, &list) - if list.Executors[0].Status != "online" || !list.Executors[0].Ready || list.Executors[0].Version != "operator-test" { + if list.Executors[0].Status != "online" || !list.Executors[0].Ready || list.Executors[0].Version != "operator-test" || list.Executors[0].Admission != wire.AdmissionReady { t.Fatalf("connected executor not ready: %+v", list) } + if err := service.WriteMaintenance(switchFile, "operator maintenance", time.Now()); err != nil { + t.Fatal(err) + } + operatorRequest(t, f, alice, http.MethodGet, "/operator/executors", nil, 200, &list) + if list.Executors[0].Admission != wire.AdmissionMaintenance || list.Executors[0].DrainStatus != "unknown" || !list.Executors[0].Ready { + t.Fatalf("dispatcher maintenance conflated with executor drain: %+v", list) + } + operatorRequest(t, f, bob, http.MethodGet, "/operator/executors", nil, 200, &list) + if len(list.Executors) != 0 { + t.Fatalf("maintenance exposed another account's inventory: %+v", list) + } + if _, err := service.ClearMaintenance(switchFile); err != nil { + t.Fatal(err) + } + operatorRequest(t, f, alice, http.MethodGet, "/operator/executors", nil, 200, &list) + if list.Executors[0].Admission != wire.AdmissionReady || list.Executors[0].DrainStatus != "unknown" { + t.Fatalf("resuming admission changed unobserved drain state: %+v", list) + } } func TestOwnedExecutorRefusalsAndQuota(t *testing.T) { From 2e210bc58ba173f55d6d1837102907910e0ba9b4 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 14:01:54 +0200 Subject: [PATCH 2/4] Require signed artifacts and a bounded restore in the external pilot --- docs/operations/external-pilot.md | 105 +++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/docs/operations/external-pilot.md b/docs/operations/external-pilot.md index bc609126..54bb87ce 100644 --- a/docs/operations/external-pilot.md +++ b/docs/operations/external-pilot.md @@ -5,32 +5,35 @@ project installs and enrolls an executor on a Linux amd64 machine they administer, using only the published documents, and reports what they observe. A container or host operated by the project does not qualify. -The pilot starts only after the project names the release, the dispatcher, the -volunteer and the measurement destinations. The volunteer's machine connects to -no dispatcher or destination before that. +The pilot starts only after the project names the exact signed release, the +dispatcher, the volunteer and host, and the measurement destinations. Agree the +measurement limits, stop condition, private support channel and restore procedure +before connecting. The volunteer's machine connects to no dispatcher or +destination before that. ## What the pilot establishes The pilot records, as observed on the volunteer's machine rather than inferred from the project's own tests: installation from the published package without a Go compiler or source checkout; enrollment of the executor with the named -dispatcher; one controlled measurement; a backup and restart from preserved -state; and the versions and capabilities the executor reports (including TLS and -SCION, where present). +dispatcher; one controlled measurement; an actual restore from a stopped-state +backup; and the versions, proxy topology and capabilities observed on that path +(including TLS and SCION, where present). Restarting the original state directory +does not demonstrate restoration from a backup. ## Prerequisites for the volunteer - A Linux amd64 machine, the supported platform in the [installation guide](../../README-install.md#supported-platforms), with a POSIX shell, GNU tar and coreutils. -- From the named release: the full archive, `install.sh` and `SHA256SUMS`. The - release must provide `dbl executor join`; v0.2.0 does not. -- Verification: the installer checks the archive against `SHA256SUMS`, which - detects changed bytes but does not identify the publisher. When the named - release is signed, verify it first as described in - [signed releases](releases.md), with the signer's public key confirmed through - the project's authenticated channel. Release signing requires administrator - setup before its first use; v0.2.0 is unsigned. +- From the named signed release: the full archive, `install.sh`, `SHA256SUMS`, + `release.json` and `release.json.sig`. The full package supplies both the CLI + and the bundled measurement; it must provide `dbl executor join`. +- Verify the signed manifest and selected installation files before execution as + described in [signed releases](releases.md), with the signer's public key + confirmed through the project's authenticated channel. Record the exact source + revision and archive digest. A checksum-only installation is insufficient for + this pilot; v0.2.0 is unsigned and lacks `dbl executor join`. - Network: the executor connects outward to the dispatcher's HTTPS API and control listeners and needs no public inbound control port ([topology](remote-deployment.md#required-topology)). @@ -73,6 +76,16 @@ commands, the output or error text, and any deviation from the documents. and the executor's reported protocols (`tcp`, `tls`, `udp`, `icmp`, `scion`), packet counter and SCION ISD-AS ([executor discovery](executor-discovery.md)). Unknown or absent values are recorded as unknown. + Record the API URL and any path prefix, the direct gRPC address and the + reverse-control address separately. For each path, record whether it is + direct, forwarded unchanged or terminated by a proxy, the observed TLS + peer/certificate identity, and any routing or timeout deviation. Successful + browser/API access alone does not demonstrate either control channel or the + executor certificate binding. Preserve native executor authentication as + described in [native TLS](remote-deployment.md#native-tls-and-executor-enrollment); + if a proxy prevents registration, record the failure instead of disabling + certificate verification. A reported SCION capability without a controlled + SCION measurement remains unvalidated. 4. **Run one controlled measurement.** With the full package, run the bundled sample against your executor: @@ -83,26 +96,70 @@ commands, the output or error text, and any deviation from the documents. Record the run ID, `dbl --dispatcher NAME status ID` and `dbl --dispatcher NAME logs ID` ([CLI reference](../cli.md)). Use only destinations agreed beforehand. -5. **Back up and restart.** Stop the executor cleanly, preserve its entire - private state directory, then start it again from that same directory with - the same configuration; no new setup token is needed - ([restart and recovery](executor-onboarding.md#restart-and-recovery)). Never - run two copies of one identity. The `dbl backup` and `dbl restore` commands - cover foreground TEST state only; to exercise them, follow - [backup and restore](backup-restore.md) with a local `dbl up` pair. Record - whether the executor returns to **Ready** and a new measurement succeeds. +5. **Back up, then restore.** Complete the stopped-state restore below. Start + only the restored copy, with the same package, configuration, certificate, + private key and executor ID; no new setup token is needed while that + certificate binding remains valid. Record whether that identity returns to + **Ready** and a newly submitted measurement completes with output. Keep + ordinary restart observations separate from the restore result. 6. **Shut down.** Stop a foreground executor normally. For a system service, use `dbl drain` and then `dbl service uninstall`, which keeps the identity and database ([managed services](services.md)). +## Restore an enrolled executor + +The volunteer and deployment operator agree and validate a filesystem +backup/restore method appropriate to the deployment before the pilot. `dbl backup` +and `dbl restore` support only [foreground TEST state](backup-restore.md); they +refuse enrolled TLS or managed-service state. A separate local TEST drill does +not establish that this external executor can be restored. + +This drill restores a fresh snapshot while the original executor stays stopped +for the entire backup-to-restore interval: no newer executor lifetime, signing +keys or work may be issued after the snapshot. It does not establish historical +backup rollback or rollback of TESLA signing state. If the source executor has +resumed since the snapshot, stop and review recovery with the operator rather +than reuse potentially stale signing state. + +1. Stop the executor cleanly and confirm its process has exited. For managed + services, stop the selected unit and prevent automatic restart during the + restore. Record the package/source identity, executor ID, canonical state + path and service account. Preserve the full private state directory and any + configuration, TLS files or other runtime dependencies referenced outside it. + Preserve ownership and permissions, and retain a private inventory and hashes + with the backup. Keep the backup off the public tracker. +2. Keep the original stopped state separately for recovery. Restore the saved + files into a new private staging directory using the agreed backup method; + check the complete inventory, hashes, ownership and permissions before use. + An incomplete copy, missing key or incompatible package stops the drill. +3. With all writers still stopped, place the verified restored files at the + original configured paths, retaining the original files separately. Managed + services require their canonical state directory; do not point them at an + arbitrary copy or silently rewrite enrollment configuration. Restore any + external dependencies to the paths the unchanged configuration names. Never + start the original and restored identities together. +4. Start the restored executor with the same signed package. Inspect registration + and the saved executor ID, then submit a fresh measurement explicitly and + verify its output. Do not replay retained work or infer that an interrupted + run completed. Record any expired certificate or changed dispatcher binding + as a recovery failure requiring the operator's decision, rather than enrolling + a new identity and counting it as a successful restore. + +Record the backup time, stop time, restore start/end, restored readiness and fresh +output time. A restored executor does not roll back the dispatcher's stored +results or prove recovery of historical packet-attribution keys. Keep those +limits explicit. If the host has no complete, agreed restore method, this part +of the pilot remains pending. + ## What the project records - Time to first result: from the start of installation to the first completed measurement. - Support effort: every contact, with its time, question, answer and the minutes spent. -- Recovery behaviour after the restart from the original state directory in - step 5. +- Recovery behaviour after starting the restored backup in step 5, including + the observed backup age, restore duration, preserved identity and time to fresh + output. Report restart-only evidence separately. - Deviations from the documents and documentation defects. The project keeps the pilot record privately with the deployment's From 58352ce0e2c7b30b2f45d8d9aae026acaf28566c Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 14:03:42 +0200 Subject: [PATCH 3/4] Version owner admission observations as API 1.20 Identify the additive owner admission and unknown drain fields with a new API minor version so clients can detect their availability. GitHub issue #215. --- api/openapi.yaml | 2 +- api/spec.go | 5 +++-- docs/api.md | 2 +- docs/versions.md | 6 ++++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index d6daf9ec..a43e4396 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: title: "Debuglet dispatcher HTTP API" - version: "1.19" + version: "1.20" description: "Wire contract of the dispatcher's public HTTP API. Bandwidth values are bits per second, timeouts are milliseconds, TESLA delays are seconds, TESLA anchor timestamps are Unix nanoseconds and every other timestamp is Unix seconds unless a field says otherwise. A client states the contract version it requires with the Debuglet-API-Version request header, declared on every operation below; every response carries the version this dispatcher implements in the same header. Requests are authenticated with a server-issued session: a native client presents it as an Authorization bearer token, a browser client in the session_token cookie plus the X-Debuglet-CSRF header on state-changing requests. Each operation states its own 401 and 403. Access to another account's run or payment order answers 404, so a response is no existence oracle. A dispatcher explicitly configured for the local development profile additionally serves the unauthenticated routes below as its own local operator. See docs/api.md for the access matrix and the compatibility and deprecation policy." servers: - url: "/" diff --git a/api/spec.go b/api/spec.go index 376e538d..305f8027 100644 --- a/api/spec.go +++ b/api/spec.go @@ -15,7 +15,7 @@ var OpenAPI []byte const ( // Version is the HTTP API contract version as major.minor. It equals the // info.version field of OpenAPI. - Version = "1.19" + Version = "1.20" // Major is the compatibility number of Version. Clients that require a // different major version are answered with an explicit incompatibility // error instead of a best-effort response. @@ -45,7 +45,8 @@ const ( // /executors and its status filter. // 1.18 added server-assisted verification and receipt keys. // 1.19 added recorded destination policies: deny, reason, expiry and their listing. - Minor = 19 + // 1.20 added admission and unavailable drain status to account-owned executors. + Minor = 20 // VersionHeader carries the contract version a client requires on requests // and the version the dispatcher implements on responses. VersionHeader = "Debuglet-API-Version" diff --git a/docs/api.md b/docs/api.md index e95a01c7..32e2c4eb 100644 --- a/docs/api.md +++ b/docs/api.md @@ -145,7 +145,7 @@ API 1.10 adds authenticated `GET` and `POST /operator/executors`, plus operations scoped to the caller's machines; they do not grant dispatcher-wide operator privileges. Inventory includes pending and offline machines. -Each inventory entry separately reports `admission`: `ready` (subject to normal +API 1.20 adds `admission` to each inventory entry: `ready` (subject to normal admission checks), `maintenance` (the dispatcher has paused new submissions), or `offline` (no ready control session, including pending enrollment). `ready` and `status` continue to describe the control connection even during dispatcher diff --git a/docs/versions.md b/docs/versions.md index f69767a1..02f7bf8f 100644 --- a/docs/versions.md +++ b/docs/versions.md @@ -111,3 +111,9 @@ API 1.17 adds RIPE Atlas-style `status`, `status_since`, `first_connected`, `las Dispatcher schema 24 stores the executor status history in `probe_status` and `probe_addresses`, filling `probe_status` from the recorded TESLA chains. The executor listing reads it, so this build requires schema 24; upgrade explicitly (`make deploy-upgrade-db`, or `-upgrade-database`) before starting it. API 1.19 adds recorded destination policies: `PATCH /destination` accepts `denied`, `reason` and `expires_at`, and `GET /destinations` lists the current policy of each destination. Dispatcher schema 26 stores their append-only history; upgrade explicitly before starting this build. The control protocol gains the optional `DestinationLimit.denied` field without a version change; an executor that does not know it still applies the zero limit sent with it, and an executor that knows it closes the active sockets to the destination. + +API 1.20 adds `admission` and `drain_status` to account-owned executor inventory. +Admission reports the current control readiness and dispatcher maintenance. +Drain status is explicitly unknown because the existing control protocol does +not observe joined host service shutdown. No schema or control-protocol change +is required. Older servers omit these observations; clients show them as unavailable. From 9faf37375bf32ac48ebe892d629965df7c829d34 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Wed, 7 Oct 2026 14:16:18 +0200 Subject: [PATCH 4/4] Preserve caller cancellation cause during concurrent runtime close --- internal/executor/debuglet/debuglet.go | 11 ++++- internal/executor/debuglet/lifecycle_test.go | 49 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/internal/executor/debuglet/debuglet.go b/internal/executor/debuglet/debuglet.go index e9b5f9e6..913cd09d 100644 --- a/internal/executor/debuglet/debuglet.go +++ b/internal/executor/debuglet/debuglet.go @@ -560,6 +560,7 @@ func (w *chanWriter) Write(p []byte) (int, error) { // back through outputCh, and returns any execution error. // The outputCh channel is closed when the debuglet finishes execution. func (d *Debuglet) Run(ctx context.Context, outputCh chan<- []byte, args []string) (err error) { + caller := ctx ctx, cancel := context.WithCancelCause(ctx) defer cancel(nil) writer := &chanWriter{ctx: ctx, ch: outputCh} @@ -583,7 +584,15 @@ func (d *Debuglet) Run(ctx context.Context, outputCh chan<- []byte, args []strin d.mu.Unlock() return errors.New("runtime is not initialized") } - d.running, d.runCancel = true, cancel + d.running = true + d.runCancel = func(cause error) { + // The caller's Done can wake a Close watcher before cancellation has + // propagated to this child. Preserve that already established cause. + if callerCause := context.Cause(caller); callerCause != nil { + cause = callerCause + } + cancel(cause) + } d.mu.Unlock() // Runtime.Close disposes WASI resources that guest host calls still use. // Keep ownership through both instantiation and module cleanup; Close diff --git a/internal/executor/debuglet/lifecycle_test.go b/internal/executor/debuglet/lifecycle_test.go index ac193258..f2716832 100644 --- a/internal/executor/debuglet/lifecycle_test.go +++ b/internal/executor/debuglet/lifecycle_test.go @@ -283,6 +283,55 @@ func TestCloseDuringRunDefersRuntimeDisposal(t *testing.T) { } } +// A parent's Done can wake its owner's Close watcher before cancellation has +// reached every child. Hold propagation to reproduce that ordering without +// relying on the scheduler, while retaining the caller's original cause. +type delayedCancellationContext struct { + context.Context + done chan struct{} + propagate chan struct{} +} + +func (c *delayedCancellationContext) Done() <-chan struct{} { return c.done } + +func (c *delayedCancellationContext) AfterFunc(f func()) func() bool { + return context.AfterFunc(c.Context, func() { + <-c.propagate + f() + }) +} + +func TestClosePreservesCallerCauseBeforeChildPropagation(t *testing.T) { + parent, cancel := context.WithCancelCause(context.Background()) + ctx := &delayedCancellationContext{Context: parent, done: make(chan struct{}), propagate: make(chan struct{})} + entered, finished := make(chan struct{}), make(chan struct{}) + rt := &closeErrorRuntime{instantiate: func(child context.Context) (api.Module, error) { + close(entered) + <-child.Done() + return nil, context.Cause(child) + }} + deb := &Debuglet{env: &wasm.WasmEnv{Logger: zap.NewNop().Sugar()}, runtime: rt, compiled: &closeErrorCompiled{}} + var runErr error + go func() { defer close(finished); runErr = deb.Run(ctx, make(chan []byte), nil) }() + t.Cleanup(func() { + cancel(context.Canceled) + close(ctx.propagate) + _ = deb.Close(context.Background()) + joinRuntimeTest(t, finished) + }) + joinRuntimeTest(t, entered) + cause := errors.New("operator cancellation before child propagation") + cancel(cause) + close(ctx.done) + if err := deb.Close(context.Background()); err != nil { + t.Fatal(err) + } + joinRuntimeTest(t, finished) + if !errors.Is(runErr, cause) { + t.Fatalf("Close replaced the caller's cancellation cause: %v", runErr) + } +} + func TestRunRefusesRuntimeStillInitializing(t *testing.T) { rt := &closeErrorRuntime{instantiate: func(context.Context) (api.Module, error) { t.Fatal("Run entered runtime before initialization returned")