Bind deployed executors before requiring client certificates

CHANGELOG.md

@@ -89,6 +89,32 @@ changes; the linked API and deployment documentation contains operational detail
   listed as unconfirmed. Dispatcher
   databases require the explicit upgrade to schema 26. See
   `docs/operations/configuration.md#destination-limits-and-opt-outs`.
+- `debuglet-dispatcher -bind-executor EXECUTOR_ID -bind-certificate PATH`
+  binds an executor ID to a client certificate the administrator issued,
+  without an enrollment token (#415). It verifies the PEM certificate against
+  the configured `tls.ca_file` for client authentication, requires the
+  executor ID as its common name and refuses a file holding a private key. It
+  records the leaf's SHA-256 fingerprint, the value an enforcing dispatcher
+  admits the executor by; binding the same certificate again changes nothing,
+  and a different one replaces that executor ID's binding only.
+- Ansible: `executor_enrollment_token`, a per-host secret rendered as
+  `[credentials] enrollment_token` only when set, for an executor the
+  deployment does not bind from a certificate it holds (#415).
+
+### Fixed
+- Turning on `dispatcher_require_client_cert` no longer disconnects every
+  executor deployed with `generate-certs.sh` and `deploy-certs.yml` (#415).
+  Enforcement admits an executor only over the certificate bound to its ID,
+  and nothing bound those certificates. With
+  `dispatcher_bind_inventory_executors` (default on), `site.yml`,
+  `update-config.yml` and `deploy-certs.yml` now copy each inventory
+  executor's public `client.crt` to the dispatcher and bind it as the service
+  account before the dispatcher runs with the requirement, and
+  `deploy-certs.yml` re-binds a certificate `generate-certs.sh` reissued. The
+  preflight refuses to enable enforcement while an inventory executor has
+  neither a certificate to bind nor an enrollment token. `deploy/README.md`
+  and the operations guides no longer suggest that a certificate from the
+  deployment CA is enough.
 
 ## [0.3.0-rc.1] - 2026-10-06
 

cmd/dispatcher/bind_executor_test.go

@@ -0,0 +1,196 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2026 ETH Zurich
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/netsec-ethz/debuglet/internal/dispatcher/enrollment"
+	"github.com/netsec-ethz/debuglet/internal/sqlitedb"
+	"github.com/netsec-ethz/debuglet/internal/storagecheck"
+	"github.com/netsec-ethz/debuglet/internal/testtls"
+)
+
+// An executor deployed with a certificate the administrator issued has no
+// token to enrol with, so the deployment binds it on the dispatcher host. The
+// binding is what the enforcing dispatcher admits the executor by.
+func TestBindExecutorAdmitsAnAdministratorIssuedCertificate(t *testing.T) {
+	const executorID = "5fe02882-0410-416c-9935-235090bcba0d"
+	dir := filepath.Join(t.TempDir(), "state")
+	if err := os.Mkdir(dir, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chmod(dir, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	path := filepath.Join(dir, "dispatcher.sqlite")
+	if err := storagecheck.BootstrapFresh(context.Background(), storagecheck.Dispatcher, path); err != nil {
+		t.Fatalf("bootstrap database: %v", err)
+	}
+	certs := t.TempDir()
+	authority, err := testtls.NewAuthority(certs, "deployment-ca")
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, err := authority.Issue(executorID, testtls.Options{Client: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The deployment copies only the public certificate to the dispatcher.
+	certFile := filepath.Join(certs, "client.crt")
+	if err := os.WriteFile(certFile, client.CertPEM, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cfg := dispatcherConfig(path)
+	cfg.TLS.Disable = false
+	cfg.TLS.CAFile = authority.CertFile
+	cfg.TLS.RequireClientCert = true
+	sum := sha256.Sum256(client.Certificate.Certificate[0])
+	fingerprint := hex.EncodeToString(sum[:])
+
+	bound := func(fingerprint string) error {
+		t.Helper()
+		db, err := sqlitedb.Open(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer db.Close()
+		return enrollment.NewStore(db).Bound(context.Background(), executorID, fingerprint)
+	}
+	if err := bound(fingerprint); !errors.Is(err, enrollment.ErrNotEnrolled) {
+		t.Fatalf("before binding: %v", err)
+	}
+
+	var out bytes.Buffer
+	if err := administerBinding(context.Background(), cfg, executorID, certFile, &out); err != nil {
+		t.Fatalf("bind: %v", err)
+	}
+	if want := "executor " + executorID + " is now bound to certificate sha256:" + fingerprint; !strings.Contains(out.String(), want) {
+		t.Fatalf("output %q, want %q", out.String(), want)
+	}
+	if err := bound(fingerprint); err != nil {
+		t.Fatalf("the bound certificate is refused: %v", err)
+	}
+
+	// Running the deployment again changes nothing, and says so.
+	out.Reset()
+	if err := administerBinding(context.Background(), cfg, executorID, certFile, &out); err != nil {
+		t.Fatalf("bind again: %v", err)
+	}
+	if !strings.Contains(out.String(), "is already bound") {
+		t.Fatalf("repeated binding printed %q", out.String())
+	}
+
+	// A reissued certificate replaces the binding, so the executor that
+	// installs it is admitted and the old certificate is not.
+	reissued, err := authority.Issue(executorID, testtls.Options{Client: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(certFile, reissued.CertPEM, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	out.Reset()
+	if err := administerBinding(context.Background(), cfg, executorID, certFile, &out); err != nil {
+		t.Fatalf("rebind: %v", err)
+	}
+	if !strings.Contains(out.String(), "replacing sha256:"+fingerprint) {
+		t.Fatalf("rebinding printed %q", out.String())
+	}
+	if err := bound(fingerprint); !errors.Is(err, enrollment.ErrWrongNode) {
+		t.Fatalf("the replaced certificate: %v", err)
+	}
+
+	// Refusals record nothing.
+	other, err := testtls.NewAuthority(t.TempDir(), "other-ca")
+	if err != nil {
+		t.Fatal(err)
+	}
+	foreign, err := other.Issue(executorID, testtls.Options{Client: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	foreignFile := filepath.Join(certs, "foreign.crt")
+	if err := os.WriteFile(foreignFile, foreign.CertPEM, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	for name, run := range map[string]func() error{
+		"another authority": func() error {
+			return administerBinding(context.Background(), cfg, executorID, foreignFile, &out)
+		},
+		"another executor ID": func() error {
+			return administerBinding(context.Background(), cfg, "1144ad6e-2c14-4e5c-ab72-c05a8e8770f2", certFile, &out)
+		},
+		"a private key": func() error {
+			return administerBinding(context.Background(), cfg, executorID, reissued.KeyFile, &out)
+		},
+		"a missing file": func() error {
+			return administerBinding(context.Background(), cfg, executorID, filepath.Join(certs, "absent.crt"), &out)
+		},
+		"no authority configured": func() error {
+			unconfigured := *cfg
+			unconfigured.TLS.CAFile = ""
+			return administerBinding(context.Background(), &unconfigured, executorID, certFile, &out)
+		},
+		"an absent database": func() error {
+			absent := *cfg
+			absent.Database.Path = filepath.Join(dir, "absent.sqlite")
+			return administerBinding(context.Background(), &absent, executorID, certFile, &out)
+		},
+	} {
+		if err := run(); err == nil {
+			t.Errorf("%s was bound", name)
+		}
+	}
+	sum = sha256.Sum256(reissued.Certificate.Certificate[0])
+	if err := bound(hex.EncodeToString(sum[:])); err != nil {
+		t.Fatalf("a refused binding changed the recorded one: %v", err)
+	}
+}
+
+// The two flags name one binding, so either alone is refused before any
+// configuration or database is read, and so is combining it with a check.
+func TestBindExecutorFlagsGoTogether(t *testing.T) {
+	if os.Getenv("DEBUGLET_BIND_COMMAND_TEST") == "dispatcher" {
+		index := slices.Index(os.Args, "--")
+		os.Args = append([]string{os.Args[0]}, os.Args[index+1:]...)
+		flag.CommandLine = flag.NewFlagSet("dispatcher", flag.ExitOnError)
+		main()
+		return
+	}
+	executable, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+	absent := filepath.Join(t.TempDir(), "absent.toml")
+	for _, tc := range []struct {
+		args []string
+		want string
+	}{
+		{[]string{"-config", absent, "-bind-executor", "executor"}, "must be given together"},
+		{[]string{"-config", absent, "-bind-certificate", "client.crt"}, "must be given together"},
+		{[]string{"-config", absent, "-check-database", "-bind-executor", "executor", "-bind-certificate", "client.crt"}, "cannot be combined"},
+	} {
+		ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+		command := exec.CommandContext(ctx, executable, append([]string{"-test.run=^TestBindExecutorFlagsGoTogether$", "--"}, tc.args...)...)
+		command.Env = append(os.Environ(), "DEBUGLET_BIND_COMMAND_TEST=dispatcher")
+		output, err := command.CombinedOutput()
+		cancel()
+		if err == nil || !strings.Contains(string(output), tc.want) {
+			t.Errorf("%v: %v, output %q, want a refusal mentioning %q", tc.args, err, output, tc.want)
+		}
+	}
+}

cmd/dispatcher/init_database_test.go

@@ -43,7 +43,7 @@ func TestInitializeDatabaseCommand(t *testing.T) {
 		t.Fatal(err)
 	}
 	path := filepath.Join(directory, "dispatcher.sqlite")
-	for _, extra := range [][]string{{"-version"}, {"-upgrade-database"}, {"-check-database"}, {"-accept-data-loss"}, {"-config", "unused"}, {"-ready-file", "unused"}, {"-grant-operator", "unused"}, {"-revoke-operator", "unused"}, {"-enroll-executor", "unused"}, {"-revoke-executor", "unused"}, {"unexpected"}} {
+	for _, extra := range [][]string{{"-version"}, {"-upgrade-database"}, {"-check-database"}, {"-accept-data-loss"}, {"-config", "unused"}, {"-ready-file", "unused"}, {"-grant-operator", "unused"}, {"-revoke-operator", "unused"}, {"-enroll-executor", "unused"}, {"-revoke-executor", "unused"}, {"-bind-executor", "unused", "-bind-certificate", "unused"}, {"unexpected"}} {
 		run(false, append([]string{"-init-database", path}, extra...)...)
 		if _, err := os.Lstat(path); !os.IsNotExist(err) {
 			t.Fatalf("refused arguments created database: %v", err)

cmd/dispatcher/main.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -41,6 +42,7 @@ import (
 	"github.com/netsec-ethz/debuglet/internal/readiness"
 	"github.com/netsec-ethz/debuglet/internal/sqlitedb"
 	"github.com/netsec-ethz/debuglet/internal/storagecheck"
+	"github.com/netsec-ethz/debuglet/internal/tlsfiles"
 
 	"github.com/google/uuid"
 
@@ -55,6 +57,8 @@ func main() {
 	revoke := flag.String("revoke-operator", "", "Return the account with this UUID to the ordinary role in the configured database, then exit")
 	enroll := flag.String("enroll-executor", "", "Create a single-use enrollment token for this executor ID in the configured database, print it once, then exit")
 	unenroll := flag.String("revoke-executor", "", "Delete the node credential enrolled for this executor ID in the configured database, then exit")
+	bind := flag.String("bind-executor", "", "Bind this executor ID to the client certificate named by -bind-certificate in the configured database, then exit")
+	bindCertificate := flag.String("bind-certificate", "", "With -bind-executor, the PEM client certificate issued for that executor by the authority in tls.ca_file")
 	initDatabase := flag.String("init-database", "", "Create a new database at this path, then exit; its parent must be a private directory owned by this user")
 	upgrade := flag.Bool("upgrade-database", false, "Apply the packaged migrations to the configured database, then exit. Stop the daemon and back the file up first")
 	checkDatabase := flag.Bool("check-database", false, "Report whether the configured database is supported by this build, then exit; exit status 3 means it needs the upgrade, 4 that the upgrade drops recorded data")
@@ -120,10 +124,14 @@ func main() {
 		return
 	}
 
-	if *checkDatabase && (*upgrade || *acceptDataLoss || *grant != "" || *revoke != "" || *enroll != "" || *unenroll != "") {
+	if *checkDatabase && (*upgrade || *acceptDataLoss || *grant != "" || *revoke != "" || *enroll != "" || *unenroll != "" || *bind != "") {
 		fmt.Fprintln(os.Stderr, "dispatcher: -check-database cannot be combined with another administration flag")
 		os.Exit(1)
 	}
+	if (*bind == "") != (*bindCertificate == "") {
+		fmt.Fprintln(os.Stderr, "dispatcher: -bind-executor and -bind-certificate must be given together")
+		os.Exit(1)
+	}
 	if *acceptDataLoss && !*upgrade {
 		fmt.Fprintln(os.Stderr, "dispatcher: -accept-data-loss is only valid with -upgrade-database")
 		os.Exit(1)
@@ -164,7 +172,7 @@ func main() {
 	// A database is upgraded only when its operator asks for it, never at
 	// start: a normal start refuses an outdated schema instead.
 	if *upgrade {
-		if *grant != "" || *revoke != "" || *enroll != "" || *unenroll != "" {
+		if *grant != "" || *revoke != "" || *enroll != "" || *unenroll != "" || *bind != "" {
 			fmt.Fprintln(os.Stderr, "dispatcher: -upgrade-database cannot be combined with another administration flag")
 			os.Exit(1)
 		}
@@ -195,6 +203,10 @@ func main() {
 	// Role administration is deliberately not an HTTP operation: the operator
 	// role is granted on the dispatcher host, by whoever already controls the
 	// database, and never by anything reachable over the network.
+	if *bind != "" && (*grant != "" || *revoke != "" || *enroll != "" || *unenroll != "") {
+		fmt.Fprintln(os.Stderr, "dispatcher: -bind-executor cannot be combined with another administration flag")
+		os.Exit(1)
+	}
 	if *grant != "" || *revoke != "" {
 		if err := administerRole(context.Background(), cfg, *grant, *revoke); err != nil {
 			fmt.Fprintf(os.Stderr, "dispatcher: %v\n", err)
@@ -212,6 +224,15 @@ func main() {
 		}
 		return
 	}
+	// A certificate the administrator issued is bound the same way: the
+	// deployment records the executors whose client certificates it issued.
+	if *bind != "" {
+		if err := administerBinding(context.Background(), cfg, *bind, *bindCertificate, os.Stdout); err != nil {
+			fmt.Fprintf(os.Stderr, "dispatcher: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
 
 	logger := daemonlog.New(cfg.Logging.LogLevel, cfg.Logging.JSONLogs)
 	defer logger.Sync()
@@ -688,6 +709,75 @@ func administerEnrollment(ctx context.Context, cfg *config.DispatcherConfig, enr
 	return nil
 }
 
+// administerBinding binds executorID to the client certificate in certPath,
+// which the administrator issued from the authority in tls.ca_file, and
+// returns. It is how executors deployed with administrator-issued
+// certificates are admitted once tls.require_client_cert is on: no token is
+// involved, because whoever controls this database already vouches for them.
+// The certificate is verified as the transport would verify it, so a binding
+// that could never admit its executor is refused rather than recorded.
+// Binding the certificate already bound changes nothing; a different one
+// replaces that executor ID's binding and no other.
+func administerBinding(ctx context.Context, cfg *config.DispatcherConfig, executorID, certPath string, out io.Writer) error {
+	if cfg.TLS.CAFile == "" {
+		return errors.New("tls.ca_file is not set, so there is no authority to verify the certificate against; configure the authority executor certificates are checked with first")
+	}
+	now := time.Now()
+	roots, err := tlsfiles.TrustRoots("tls.ca_file", cfg.TLS.CAFile, now)
+	if err != nil {
+		return err
+	}
+	certPEM, err := readBounded(certPath, enrollment.MaxCertificateBytes)
+	if err != nil {
+		return fmt.Errorf("-bind-certificate: %w", err)
+	}
+	fingerprint, err := enrollment.CertificateFingerprint(executorID, certPEM, roots, now)
+	if err != nil {
+		return fmt.Errorf("-bind-certificate %s: %w", certPath, err)
+	}
+	if err := storagecheck.Check(ctx, storagecheck.Dispatcher, cfg.Database.Path); err != nil {
+		return err
+	}
+	db, err := sqlitedb.Open(cfg.Database.Path)
+	if err != nil {
+		return fmt.Errorf("open database: %w", err)
+	}
+	defer db.Close()
+	binding, err := enrollment.NewStore(db).Bind(ctx, executorID, fingerprint)
+	if err != nil {
+		return err
+	}
+	if !cfg.TLS.RequireClientCert {
+		fmt.Fprintln(os.Stderr, "dispatcher: tls.require_client_cert is not set, so this dispatcher does not check this binding until it is")
+	}
+	switch {
+	case !binding.Changed:
+		_, err = fmt.Fprintf(out, "executor %s is already bound to certificate sha256:%s\n", executorID, fingerprint)
+	case binding.Previous == "":
+		_, err = fmt.Fprintf(out, "executor %s is now bound to certificate sha256:%s\n", executorID, fingerprint)
+	default:
+		_, err = fmt.Fprintf(out, "executor %s is now bound to certificate sha256:%s, replacing sha256:%s\n", executorID, fingerprint, binding.Previous)
+	}
+	return err
+}
+
+// readBounded reads a file of at most limit bytes.
+func readBounded(path string, limit int64) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	data, err := io.ReadAll(io.LimitReader(f, limit+1))
+	if err != nil {
+		return nil, err
+	}
+	if int64(len(data)) > limit {
+		return nil, fmt.Errorf("%s exceeds %d bytes", path, limit)
+	}
+	return data, nil
+}
+
 // localDevelopmentProfile reports whether the HTTP API serves its local
 // development profile. It takes both the operator's explicit opt-in and an
 // environment this daemon recognises as local: neither alone turns