From 88d01ef0e3b9a69b4e7f540d8231f118d02a7b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Schottst=C3=A4dt?= Date: Tue, 27 Oct 2026 01:17:07 +0100 Subject: [PATCH 1/2] feat(runtime): preserve and apply exact releases Preserve installed release identity through backup/restore and accepted updates. Cache release descriptors from pull results and enforce the expected installed digest before application. Stage restoration with rollback, retain ownership/file protections, and remove the standalone semantic validator. Fix the running-backup fixture so its queue stops before temporary state cleanup. --- api/openapi.yaml | 41 +- apps/druid/adapters/cli/client/update.go | 11 +- apps/druid/adapters/cli/pull.go | 4 +- apps/druid/adapters/cli/push.go | 23 +- apps/druid/adapters/cli/push_test.go | 27 ++ .../cli/worker_glob_protection_test.go | 49 +++ apps/druid/adapters/cli/worker_ownership.go | 84 +++++ .../adapters/cli/worker_ownership_test.go | 48 +++ .../adapters/cli/worker_permissions_test.go | 58 +++ apps/druid/adapters/cli/worker_pull.go | 313 ++++++++++++---- apps/druid/adapters/cli/worker_push.go | 4 +- .../druid/adapters/cli/worker_release_test.go | 96 +++++ apps/druid/adapters/cli/worker_test.go | 261 ++++++++++--- .../adapters/daemonclient/openapi_client.go | 5 +- apps/druid/adapters/http/handlers/routes.go | 2 + .../adapters/http/handlers/routes_test.go | 64 ++++ .../adapters/http/handlers/scroll_handler.go | 23 +- apps/druid/core/services/runtime_access.go | 50 ++- .../services/runtime_backup_failure_test.go | 36 ++ .../runtime_command_maintenance_test.go | 94 +++++ apps/druid/core/services/runtime_lifecycle.go | 11 + .../core/services/runtime_materialization.go | 16 - apps/druid/core/services/runtime_release.go | 82 ++++ .../core/services/runtime_release_test.go | 175 +++++++++ .../services/runtime_restore_failure_test.go | 69 ++++ .../core/services/runtime_session_cache.go | 14 + .../core/services/runtime_session_commands.go | 24 +- .../core/services/runtime_session_queue.go | 40 +- .../core/services/runtime_session_runtime.go | 16 +- .../druid/core/services/runtime_supervisor.go | 41 +- .../core/services/runtime_supervisor_test.go | 353 +++++++++++++++--- apps/druid/core/services/runtime_update.go | 42 ++- .../core/services/runtime_update_test.go | 53 +++ docs/runtime-lifecycle-verification.md | 72 ++++ internal/api/generated.go | 246 +++++++++--- internal/core/domain/runtime_scroll.go | 41 +- internal/core/ports/services_ports.go | 4 +- internal/core/services/registry/oci.go | 45 ++- internal/core/services/registry/oci_test.go | 110 ++++++ .../services/registry/transfer_options.go | 8 + internal/runtime/backend_factory_test.go | 2 +- internal/runtime/docker/state_store.go | 21 +- internal/runtime/docker/state_store_test.go | 44 +++ internal/runtime/docker/workers.go | 55 ++- internal/runtime/kubernetes/resources.go | 22 +- internal/runtime/kubernetes/resources_test.go | 25 +- internal/runtime/kubernetes/state_store.go | 93 ++--- .../runtime/kubernetes/state_store_test.go | 32 +- internal/runtime/kubernetes/workers.go | 4 +- internal/utils/copy.go | 102 +++++ internal/utils/fs.go | 6 +- test/integration/docker/docker_cli_test.go | 6 + test/integration/docker/permissions_test.go | 109 ++++++ test/integration/internal/e2e/harness.go | 4 +- .../kubernetes/kubernetes_cli_test.go | 72 ++++ .../kubernetes/registry_resolution_test.go | 44 +++ test/mock/services.go | 8 +- 57 files changed, 2971 insertions(+), 433 deletions(-) create mode 100644 apps/druid/adapters/cli/push_test.go create mode 100644 apps/druid/adapters/cli/worker_glob_protection_test.go create mode 100644 apps/druid/adapters/cli/worker_ownership.go create mode 100644 apps/druid/adapters/cli/worker_ownership_test.go create mode 100644 apps/druid/adapters/cli/worker_permissions_test.go create mode 100644 apps/druid/adapters/cli/worker_release_test.go create mode 100644 apps/druid/core/services/runtime_backup_failure_test.go create mode 100644 apps/druid/core/services/runtime_command_maintenance_test.go create mode 100644 apps/druid/core/services/runtime_release.go create mode 100644 apps/druid/core/services/runtime_release_test.go create mode 100644 apps/druid/core/services/runtime_restore_failure_test.go create mode 100644 apps/druid/core/services/runtime_update_test.go create mode 100644 docs/runtime-lifecycle-verification.md create mode 100644 internal/core/services/registry/transfer_options.go create mode 100644 internal/utils/copy.go create mode 100644 test/integration/docker/permissions_test.go create mode 100644 test/integration/kubernetes/registry_resolution_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index 32be8a07..594134ee 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -209,10 +209,16 @@ components: UpdateScrollRequest: type: object + required: [artifact] properties: artifact: type: string - description: Optional target artifact. If omitted, the daemon refreshes the runtime's current artifact. + pattern: '^[^@\s]+@sha256:[a-f0-9]{64}$' + description: Explicitly accepted immutable target. Missing or mutable tag references are rejected before stopping the runtime. + expected_installed_digest: + type: string + pattern: '^sha256:[a-f0-9]{64}$' + description: Optional precondition checked against the actual installed descriptor under the maintenance lock. registry_credentials: type: array items: @@ -482,6 +488,33 @@ paths: '404': description: Runtime scroll not found + /api/v1/scrolls/{id}/release: + get: + operationId: getInstalledRelease + tags: [scrolls] + summary: Read the installed release descriptor from the runtime volume + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + '200': + description: Installed canonical repository and last successful release digest + content: + application/json: + schema: + type: object + required: [repository, digest] + properties: + repository: + type: string + digest: + type: string + '404': + description: Runtime scroll not found + /api/v1/scrolls/{id}/start: post: operationId: startScroll @@ -532,7 +565,7 @@ paths: schema: type: string requestBody: - required: false + required: true content: application/json: schema: @@ -544,6 +577,10 @@ paths: application/json: schema: $ref: '#/components/schemas/RuntimeScroll' + '400': + description: An explicitly accepted SHA256 artifact reference is required + '409': + description: Installed release changed since the update was checked '404': description: Runtime scroll not found diff --git a/apps/druid/adapters/cli/client/update.go b/apps/druid/adapters/cli/client/update.go index 45e8f969..3fb36cf5 100644 --- a/apps/druid/adapters/cli/client/update.go +++ b/apps/druid/adapters/cli/client/update.go @@ -3,14 +3,11 @@ package client import "github.com/spf13/cobra" var UpdateCommand = &cobra.Command{ - Use: "update [artifact]", - Short: "Update a daemon-managed scroll runtime", - Args: cobra.RangeArgs(1, 2), + Use: "update ", + Short: "Apply an explicitly accepted immutable Scroll revision", + Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - artifact := "" - if len(args) == 2 { - artifact = args[1] - } + artifact := args[1] daemon, err := runtimeDaemonClient() if err != nil { return err diff --git a/apps/druid/adapters/cli/pull.go b/apps/druid/adapters/cli/pull.go index 16c8e3df..6349731a 100644 --- a/apps/druid/adapters/cli/pull.go +++ b/apps/druid/adapters/cli/pull.go @@ -7,6 +7,7 @@ import ( ) var pullNoData bool +var pullPreserveReleaseManifest bool var PullCommand = &cobra.Command{ Use: "pull [dir]", @@ -21,7 +22,7 @@ var PullCommand = &cobra.Command{ registryClient := registry.NewOciClient(loadRegistryStore()) - err := registryClient.PullSelective(dir, artifact, !pullNoData, nil) + err := registryClient.PullSelectiveWithOptions(dir, artifact, !pullNoData, nil, registry.TransferOptions{PreserveReleaseManifest: pullPreserveReleaseManifest}) if err != nil { logger.Log().Error("Failed to pull from registry") return err @@ -35,4 +36,5 @@ var PullCommand = &cobra.Command{ func init() { RootCmd.AddCommand(PullCommand) PullCommand.Flags().BoolVar(&pullNoData, "no-data", false, "Skip scroll data files") + PullCommand.Flags().BoolVar(&pullPreserveReleaseManifest, "preserve-release-manifest", false, "Keep the release manifest.json stored in a backup artifact.") } diff --git a/apps/druid/adapters/cli/push.go b/apps/druid/adapters/cli/push.go index 43fb4d77..edab2efa 100644 --- a/apps/druid/adapters/cli/push.go +++ b/apps/druid/adapters/cli/push.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" + "time" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/services/registry" @@ -22,6 +24,7 @@ var pushScrollPorts []string var pushPackMeta bool var pushSmart bool var pushCategory string +var pushPreserveReleaseManifest bool var pushDisableTarReproducible bool var PushCommand = &cobra.Command{ @@ -68,6 +71,9 @@ var PushCommand = &cobra.Command{ } overrides := map[string]string{} + if err := reproducibleCreatedAnnotation(overrides, os.Getenv("SOURCE_DATE_EPOCH")); err != nil { + return err + } if pushMinRAM != "" { overrides["gg.druid.scroll.minRam"] = pushMinRAM } @@ -96,7 +102,7 @@ var PushCommand = &cobra.Command{ overrides[fmt.Sprintf("gg.druid.scroll.port.%s", name)] = port } - _, err = ociClient.Push(fullPath, repo, tag, overrides, pushPackMeta, &scroll.File) + _, err = ociClient.PushWithOptions(fullPath, repo, tag, overrides, pushPackMeta, &scroll.File, registry.TransferOptions{PreserveReleaseManifest: pushPreserveReleaseManifest}) if err != nil { return err } @@ -106,6 +112,20 @@ var PushCommand = &cobra.Command{ }, } +// ORAS otherwise stamps the current time, making an identical CI rebuild a +// different immutable revision. Only explicitly reproducible builds override it. +func reproducibleCreatedAnnotation(annotations map[string]string, epoch string) error { + if epoch == "" { + return nil + } + seconds, err := strconv.ParseInt(epoch, 10, 64) + if err != nil || seconds < 0 || seconds > 253402300799 { + return fmt.Errorf("SOURCE_DATE_EPOCH must be a nonnegative Unix timestamp before year 10000") + } + annotations["org.opencontainers.image.created"] = time.Unix(seconds, 0).UTC().Format(time.RFC3339) + return nil +} + func init() { RootCmd.AddCommand(PushCommand) PushCommand.Flags().StringVarP(&pushMinRAM, "min-ram", "r", pushMinRAM, "Minimum RAM required to run the application. (Will be added as a manifest annotation gg.druid.scroll.minRam)") @@ -116,5 +136,6 @@ func init() { PushCommand.Flags().StringVarP(&pushImage, "image", "i", pushImage, "Image to use for the scroll. (Will be added as a manifest annotation gg.druid.scroll.image)") PushCommand.Flags().StringSliceVarP(&pushScrollPorts, "port", "p", pushScrollPorts, "Ports to expose. Format webserver=80, dns=53/udp or just minecraft (Will be added as a manifest annotation gg.druid.scroll.ports.)") PushCommand.Flags().BoolVarP(&pushPackMeta, "pack-meta", "m", pushPackMeta, "Pack the meta folder into the scroll.") + PushCommand.Flags().BoolVar(&pushPreserveReleaseManifest, "preserve-release-manifest", false, "Include the installed release manifest.json in this snapshot.") PushCommand.PersistentFlags().BoolVar(&pushDisableTarReproducible, "no-tar-reproducible", false, "Preserve file timestamps in pushed tar layers.") } diff --git a/apps/druid/adapters/cli/push_test.go b/apps/druid/adapters/cli/push_test.go new file mode 100644 index 00000000..781e136a --- /dev/null +++ b/apps/druid/adapters/cli/push_test.go @@ -0,0 +1,27 @@ +package cli + +import "testing" + +func TestExplicitSourceDateMakesReleaseCreationDeterministic(t *testing.T) { + for i := 0; i < 2; i++ { + annotations := map[string]string{} + if err := reproducibleCreatedAnnotation(annotations, "0"); err != nil { + t.Fatal(err) + } + if annotations["org.opencontainers.image.created"] != "1970-01-01T00:00:00Z" { + t.Fatal("unstable creation annotation") + } + } + annotations := map[string]string{} + if err := reproducibleCreatedAnnotation(annotations, ""); err != nil { + t.Fatal(err) + } + if len(annotations) != 0 { + t.Fatal("ordinary pushes unexpectedly changed") + } + for _, invalid := range []string{"-1", "tomorrow", "253402300800"} { + if err := reproducibleCreatedAnnotation(annotations, invalid); err == nil { + t.Fatal("invalid source date accepted") + } + } +} diff --git a/apps/druid/adapters/cli/worker_glob_protection_test.go b/apps/druid/adapters/cli/worker_glob_protection_test.go new file mode 100644 index 00000000..67c779b3 --- /dev/null +++ b/apps/druid/adapters/cli/worker_glob_protection_test.go @@ -0,0 +1,49 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWorkerUpdatePreservesGlobProtectedData(t *testing.T) { + for _, nested := range []bool{false, true} { + for _, declaration := range []string{"installed", "candidate"} { + t.Run(declaration+map[bool]string{false: "/flat", true: "/nested"}[nested], func(t *testing.T) { + root, candidate := t.TempDir(), t.TempDir() + chunks := "chunks:\n - name: config\n path: '*.cfg'\n skip_update: true\n" + directory := "" + if nested { + directory = "servers/one" + chunks = "chunks:\n - name: servers\n path: 'servers/*'\n chunks:\n - name: config\n path: '*.cfg'\n skip_update: true\n" + } + installedYAML, candidateYAML := "name: example\n", "name: example\n" + if declaration == "installed" { + installedYAML += chunks + } else { + candidateYAML += chunks + } + mustWrite(t, filepath.Join(root, "scroll.yaml"), installedYAML) + mustWrite(t, filepath.Join(candidate, "scroll.yaml"), candidateYAML) + mustWrite(t, filepath.Join(root, "data", directory, "edited.cfg"), "user edits") + mustWrite(t, filepath.Join(root, "data", directory, "removed.cfg"), "user save") + mustWrite(t, filepath.Join(root, "data", directory, "obsolete.txt"), "old release") + mustWrite(t, filepath.Join(candidate, "data", directory, "edited.cfg"), "new default") + mustWrite(t, filepath.Join(candidate, "data", directory, "absent.cfg"), "must not appear") + mustWrite(t, filepath.Join(candidate, "data", directory, "release.txt"), "new release") + if err := pullWorkerUpdate(root, candidate, nil, nil); err != nil { + t.Fatal(err) + } + assertFile(t, filepath.Join(root, "data", directory, "edited.cfg"), "user edits") + assertFile(t, filepath.Join(root, "data", directory, "removed.cfg"), "user save") + assertFile(t, filepath.Join(root, "data", directory, "release.txt"), "new release") + for _, name := range []string{"absent.cfg", "obsolete.txt"} { + if _, err := os.Stat(filepath.Join(root, "data", directory, name)); !os.IsNotExist(err) { + t.Fatalf("%s should be absent: %v", name, err) + } + } + assertFile(t, filepath.Join(root, "scroll.yaml"), candidateYAML) + }) + } + } +} diff --git a/apps/druid/adapters/cli/worker_ownership.go b/apps/druid/adapters/cli/worker_ownership.go new file mode 100644 index 00000000..0f9e30f4 --- /dev/null +++ b/apps/druid/adapters/cli/worker_ownership.go @@ -0,0 +1,84 @@ +package cli + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "strconv" + "strings" + + "github.com/highcard-dev/daemon/internal/utils" +) + +type workerDataOwner struct{ uid, gid int } + +func ownershipID(value string) (int, error) { + id, err := strconv.ParseUint(value, 10, 32) + if err != nil || id == 1<<32-1 { + return 0, fmt.Errorf("invalid ownership ID %q", value) + } + return int(id), nil +} + +// Match Docker user[:group] identity inside the worker image, including named +// users. Numeric users absent from passwd default to group 0, as Docker does. +func resolveWorkerDataOwner(value string) (*workerDataOwner, error) { + if value == "" { + return nil, nil + } + parts := strings.Split(value, ":") + if len(parts) > 2 || parts[0] == "" || len(parts) == 2 && parts[1] == "" { + return nil, fmt.Errorf("invalid new-data-owner %q", value) + } + owner := &workerDataOwner{} + uid, numericErr := ownershipID(parts[0]) + var account *user.User + var err error + if numericErr == nil { + owner.uid = uid + account, _ = user.LookupId(parts[0]) + } else { + account, err = user.Lookup(parts[0]) + if err != nil { + return nil, fmt.Errorf("resolve new-data-owner: %w", err) + } + owner.uid, err = ownershipID(account.Uid) + if err != nil { + return nil, err + } + } + if account != nil { + owner.gid, err = ownershipID(account.Gid) + if err != nil { + return nil, err + } + } + if len(parts) == 2 { + owner.gid, err = ownershipID(parts[1]) + if err != nil { + group, lookupErr := user.LookupGroup(parts[1]) + if lookupErr != nil { + return nil, fmt.Errorf("resolve new-data-owner group: %w", lookupErr) + } + owner.gid, err = ownershipID(group.Gid) + if err != nil { + return nil, err + } + } + } + return owner, nil +} + +func applyWorkerDataOwner(root string, owner *workerDataOwner) error { + if owner == nil { + return nil + } + return filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + // WalkDir and Lchown both avoid following symlinks outside the stage. + return utils.SetPathOwner(path, owner.uid, owner.gid) + }) +} diff --git a/apps/druid/adapters/cli/worker_ownership_test.go b/apps/druid/adapters/cli/worker_ownership_test.go new file mode 100644 index 00000000..43187927 --- /dev/null +++ b/apps/druid/adapters/cli/worker_ownership_test.go @@ -0,0 +1,48 @@ +package cli + +import ( + "os" + "os/user" + "path/filepath" + "testing" + + "github.com/highcard-dev/daemon/internal/utils" +) + +func TestWorkerOwnerResolution(t *testing.T) { + owner, err := resolveWorkerDataOwner("1234:2345") + if err != nil || owner.uid != 1234 || owner.gid != 2345 { + t.Fatalf("numeric owner: %+v %v", owner, err) + } + current, err := user.Current() + if err != nil { + t.Fatal(err) + } + owner, err = resolveWorkerDataOwner(current.Username) + if err != nil || owner.uid != os.Getuid() || owner.gid != os.Getgid() { + t.Fatalf("named owner: %+v %v", owner, err) + } + for _, invalid := range []string{"-1:0", ":0", "1000:", "1:2:3", "4294967295:0", "0:4294967295"} { + if _, err := resolveWorkerDataOwner(invalid); err == nil { + t.Errorf("accepted invalid owner %q", invalid) + } + } +} + +func TestWorkerOwnershipDoesNotFollowSymlinks(t *testing.T) { + root := t.TempDir() + if err := os.Symlink(filepath.Join(t.TempDir(), "absent"), filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + if err := applyWorkerDataOwner(root, &workerDataOwner{os.Getuid(), os.Getgid()}); err != nil { + t.Fatal(err) + } + // A dangling target makes any accidental target dereference fail. + target := filepath.Join(t.TempDir(), "copy") + if err := utils.CopyPath(root, target, true); err != nil { + t.Fatal(err) + } + if _, err := os.Readlink(filepath.Join(target, "link")); err != nil { + t.Fatal(err) + } +} diff --git a/apps/druid/adapters/cli/worker_permissions_test.go b/apps/druid/adapters/cli/worker_permissions_test.go new file mode 100644 index 00000000..4f1f82ae --- /dev/null +++ b/apps/druid/adapters/cli/worker_permissions_test.go @@ -0,0 +1,58 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/highcard-dev/daemon/internal/utils" +) + +func TestWorkerUpdatePreservesProtectedPermissions(t *testing.T) { + root, candidate := t.TempDir(), t.TempDir() + yaml := "name: example\nchunks:\n - name: config\n path: config\n skip_update: true\n" + mustWrite(t, filepath.Join(root, "scroll.yaml"), yaml) + mustWrite(t, filepath.Join(candidate, "scroll.yaml"), yaml) + mustWrite(t, filepath.Join(root, "data/config/settings.cfg"), "user settings") + mustWrite(t, filepath.Join(candidate, "data/config/settings.cfg"), "new defaults") + want := map[string]os.FileMode{"data/config": 0770, "data/config/settings.cfg": 0660} + for path, mode := range want { + if err := os.Chmod(filepath.Join(root, path), mode); err != nil { + t.Fatal(err) + } + } + if err := pullWorkerUpdate(root, candidate, nil, nil); err != nil { + t.Fatal(err) + } + assertFile(t, filepath.Join(root, "data/config/settings.cfg"), "user settings") + for path, mode := range want { + info, err := os.Stat(filepath.Join(root, path)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != mode { + t.Errorf("protected %s mode=%#o, want %#o", path, info.Mode().Perm(), mode) + } + } +} + +func TestProtectedDataCopyPreservesReadOnlyDirectoryPermissions(t *testing.T) { + source, target := t.TempDir(), t.TempDir() + mustWrite(t, filepath.Join(source, "config", "settings"), "user settings") + if err := os.Chmod(filepath.Join(source, "config"), 0550); err != nil { + t.Fatal(err) + } + defer os.Chmod(filepath.Join(source, "config"), 0700) + defer os.Chmod(filepath.Join(target, "config"), 0700) + if err := utils.CopyPath(source, target, true); err != nil { + t.Fatal(err) + } + assertFile(t, filepath.Join(target, "config", "settings"), "user settings") + info, err := os.Stat(filepath.Join(target, "config")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0550 { + t.Fatalf("directory mode=%#o, want 0550", info.Mode().Perm()) + } +} diff --git a/apps/druid/adapters/cli/worker_pull.go b/apps/druid/adapters/cli/worker_pull.go index 4bdab158..4dd382cc 100644 --- a/apps/druid/adapters/cli/worker_pull.go +++ b/apps/druid/adapters/cli/worker_pull.go @@ -16,6 +16,8 @@ import ( "github.com/highcard-dev/daemon/internal/core/ports" coreservices "github.com/highcard-dev/daemon/internal/core/services" "github.com/highcard-dev/daemon/internal/core/services/registry" + "github.com/highcard-dev/daemon/internal/utils" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -51,7 +53,11 @@ func init() { WorkerPullCommand.Flags().StringVar(&workerPullAction.MountPath, "root", "/scroll", "Mounted runtime root path") WorkerPullCommand.Flags().StringVar(&workerPullAction.CallbackURL, "callback-url", "", "Daemon worker callback URL") WorkerPullCommand.Flags().StringVar(&workerPullAction.TokenFile, "callback-token-file", "", "Projected ServiceAccount token file for callbacks") - WorkerPullCommand.Flags().StringVar(&workerPullMode, "mode", string(ports.RuntimeWorkerModeCreate), "Pull mode: create, update, or restore") + WorkerPullCommand.Flags().StringVar(&workerPullAction.NewDataOwner, "new-data-owner", "", "Default user[:group] for new materialized content; protected paths keep their owners") + // Older daemons pass this flag; restore mode always preserves the manifest. + WorkerPullCommand.Flags().Bool("preserve-release-manifest", false, "Deprecated compatibility flag; restore always preserves the release manifest") + WorkerPullCommand.Flags().MarkHidden("preserve-release-manifest") + WorkerPullCommand.Flags().StringVar(&workerPullMode, "mode", string(ports.RuntimeWorkerModeCreate), "Worker mode: create, update, restore, or inspect") WorkerPullCommand.MarkFlagRequired("artifact") WorkerPullCommand.MarkFlagRequired("runtime-id") } @@ -66,14 +72,18 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { if root == "" { root = "/scroll" } - oci := registry.NewOciClient(loadWorkerRegistryStore()) - digest, err := oci.ResolveDigest(action.Artifact) - if err == nil { - result.ArtifactDigest = digest + if action.Mode == ports.RuntimeWorkerModeInspect { + return inspectInstalledRelease(root) + } + owner, err := resolveWorkerDataOwner(action.NewDataOwner) + if err != nil { + result.Error = err.Error() + return result } + oci := registry.NewOciClient(loadWorkerRegistryStore()) switch action.Mode { case ports.RuntimeWorkerModeUpdate: - err = pullWorkerUpdate(root, action.Artifact, oci) + err = pullWorkerUpdate(root, action.Artifact, oci, owner) case ports.RuntimeWorkerModeRestore: err = pullWorkerRestore(root, action.Artifact, oci) default: @@ -83,6 +93,18 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { result.Error = err.Error() return result } + // Read the installed result, not the source tag (or backup artifact digest). + // Local directory Scrolls may have no release descriptor. + _, localErr := os.Stat(action.Artifact) + return readInstalledRelease(root, localErr != nil || action.Mode == ports.RuntimeWorkerModeRestore) +} + +func inspectInstalledRelease(root string) ports.RuntimeWorkerResult { + return readInstalledRelease(root, true) +} + +func readInstalledRelease(root string, requireDescriptor bool) ports.RuntimeWorkerResult { + result := ports.RuntimeWorkerResult{} scrollYAML, err := os.ReadFile(filepath.Join(root, "scroll.yaml")) if err != nil { result.Error = err.Error() @@ -93,6 +115,25 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { return result } result.ScrollYAML = string(scrollYAML) + file, err := os.Open(filepath.Join(root, "manifest.json")) + if os.IsNotExist(err) && !requireDescriptor { + return result + } + if err != nil { + result.Error = err.Error() + return result + } + defer file.Close() + var descriptor v1.Descriptor + if err := json.NewDecoder(io.LimitReader(file, 4*1024*1024)).Decode(&descriptor); err != nil { + result.Error = fmt.Sprintf("invalid installed release descriptor: %v", err) + return result + } + if descriptor.Digest.Validate() != nil || descriptor.Digest.Algorithm() != "sha256" { + result.Error = "installed release descriptor requires a valid sha256 digest" + return result + } + result.ArtifactDigest = descriptor.Digest.String() return result } @@ -141,7 +182,7 @@ func pullWorkerCreate(root string, artifact string, oci ports.OciRegistryInterfa return oci.PullSelective(root, artifact, true, nil) } -func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterface) error { +func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterface, owner *workerDataOwner) error { tmp, err := os.MkdirTemp("", "druid-worker-update-*") if err != nil { return err @@ -159,124 +200,242 @@ func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterfa return err } skipData := map[string]bool{} - collectSkipUpdatePaths(skipData, "", scroll.Chunks) - return mergePulledRoot(tmp, root, skipData) -} - -func pullWorkerRestore(root string, artifact string, oci ports.OciRegistryInterface) error { - tmp, err := os.MkdirTemp("", "druid-worker-restore-*") + installedYAML, err := os.ReadFile(filepath.Join(root, "scroll.yaml")) if err != nil { - return err + return fmt.Errorf("read installed update protection: %w", err) } - defer os.RemoveAll(tmp) - if err := coreservices.MaterializeScrollArtifact(artifact, tmp, oci, true); err != nil { - return err + installed, err := domain.NewScrollFromBytes(root, installedYAML) + if err != nil { + return fmt.Errorf("parse installed update protection: %w", err) + } + // Honor both sides of this transition, including protected chunks removed + // by the candidate. Do not rewrite the candidate's immutable Scroll metadata. + // Expand both declarations against both trees: preserve removed matches and + // preserve absence when a candidate introduces a protected matching path. + for _, base := range []string{root, tmp} { + for _, chunks := range [][]*domain.Chunks{installed.Chunks, scroll.Chunks} { + if err := collectSkipUpdatePaths(skipData, filepath.Join(base, domain.RuntimeDataDir), "", chunks); err != nil { + return err + } + } } if err := os.MkdirAll(root, 0755); err != nil { return err } - entries, err := os.ReadDir(root) + stage, err := os.MkdirTemp(root, ".druid-worker-update-stage-*") if err != nil { return err } - for _, entry := range entries { - if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil { + defer os.RemoveAll(stage) + if err := copyPath(tmp, stage); err != nil { + return err + } + if err := applyWorkerDataOwner(stage, owner); err != nil { + return err + } + if err := preserveSkippedUpdateData(root, stage, skipData); err != nil { + return err + } + return replaceRestoredRoot(root, stage) +} + +// preserveSkippedUpdateData copies only the paths a Scroll explicitly marks as +// skip_update from the installed root into the staged candidate. Candidate +// content is otherwise complete, so unprotected files absent from the +// candidate are removed when the staged root replaces the installed root. +func preserveSkippedUpdateData(root string, stage string, skipData map[string]bool) error { + for skip := range skipData { + skip = filepath.ToSlash(filepath.Clean(skip)) + if !filepath.IsLocal(skip) { + return fmt.Errorf("skip_update path must stay inside runtime data: %q", skip) + } + if skip == "." { + skip = "" + } + source := filepath.Join(root, domain.RuntimeDataDir, filepath.FromSlash(skip)) + target := filepath.Join(stage, domain.RuntimeDataDir, filepath.FromSlash(skip)) + for _, base := range []string{root, stage} { + if err := rejectSymlinkParents(base, filepath.Join(domain.RuntimeDataDir, filepath.FromSlash(skip))); err != nil { + return err + } + } + if err := os.RemoveAll(target); err != nil { + return err + } + if _, err := os.Lstat(source); os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + if err := utils.CopyPath(source, target, true); err != nil { return err } } - return copyPath(tmp, root) + return nil } -func collectSkipUpdatePaths(out map[string]bool, parent string, chunks []*domain.Chunks) { - for _, chunk := range chunks { - if chunk == nil { - continue +func rejectSymlinkParents(root, relative string) error { + parts := strings.Split(filepath.Clean(relative), string(filepath.Separator)) + current := root + for _, part := range parts[:len(parts)-1] { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if os.IsNotExist(err) { + return nil } - chunkPath := filepath.ToSlash(filepath.Clean(filepath.Join(parent, filepath.FromSlash(chunk.Path)))) - if chunkPath == "." { - chunkPath = "" + if err != nil { + return err } - if chunk.SkipUpdate { - out[chunkPath] = true + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("protected update path traverses symlink: %s", relative) } - collectSkipUpdatePaths(out, chunkPath, chunk.Chunks) } + return nil } -func mergePulledRoot(src string, dst string, skipData map[string]bool) error { - if err := os.MkdirAll(dst, 0755); err != nil { +type restoreArtifactPuller interface { + PullSelectiveWithOptions(string, string, bool, *domain.SnapshotProgress, registry.TransferOptions) error +} + +func pullWorkerRestore(root string, artifact string, oci restoreArtifactPuller) error { + if err := os.MkdirAll(root, 0755); err != nil { return err } - entries, err := os.ReadDir(src) + stage, err := os.MkdirTemp(root, ".druid-worker-restore-stage-*") if err != nil { return err } + defer os.RemoveAll(stage) + if err := oci.PullSelectiveWithOptions(stage, artifact, true, nil, registry.TransferOptions{PreserveReleaseManifest: true}); err != nil { + return err + } + return replaceRestoredRoot(root, stage) +} + +const restoreRootUnsafePrefix = "restore root may be partial:" + +func replaceRestoredRoot(root string, stage string) error { + return replaceRestoredRootWithRename(root, stage, os.Rename) +} + +// replaceRestoredRootWithRename swaps staged artifact entries into a runtime +// root without copying over a live tree. If an entry move fails, it restores +// the original entries before returning. An error with restoreRootUnsafePrefix +// means that rollback itself failed and callers must keep the runtime stopped. +func replaceRestoredRootWithRename(root string, stage string, rename func(string, string) error) error { + rollback, err := os.MkdirTemp(root, ".druid-worker-restore-rollback-*") + if err != nil { + return err + } + keepRecovery := false + defer func() { + if !keepRecovery { + _ = os.RemoveAll(rollback) + } + }() + + stageName := filepath.Base(stage) + rollbackName := filepath.Base(rollback) + entries, err := os.ReadDir(root) + if err != nil { + return err + } + original := make([]string, 0, len(entries)) for _, entry := range entries { - name := entry.Name() - srcPath := filepath.Join(src, name) - dstPath := filepath.Join(dst, name) - if name == domain.RuntimeDataDir { - if err := copyDataUpdate(srcPath, dstPath, skipData); err != nil { + if entry.Name() != stageName && entry.Name() != rollbackName { + original = append(original, entry.Name()) + } + } + stagedEntries, err := os.ReadDir(stage) + if err != nil { + return err + } + + restoreOriginal := func(installed []string) error { + for _, name := range installed { + if err := os.RemoveAll(filepath.Join(root, name)); err != nil { return err } - continue } - if err := os.RemoveAll(dstPath); err != nil { - return err + for _, name := range original { + from := filepath.Join(rollback, name) + if _, err := os.Lstat(from); os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + if err := rename(from, filepath.Join(root, name)); err != nil { + return err + } } - if err := copyPath(srcPath, dstPath); err != nil { - return err + return nil + } + + for _, entry := range original { + if err := rename(filepath.Join(root, entry), filepath.Join(rollback, entry)); err != nil { + if rollbackErr := restoreOriginal(nil); rollbackErr != nil { + keepRecovery = true + return fmt.Errorf("%s recovery files retained at %s; failed to restore original runtime after moving %s: %w", restoreRootUnsafePrefix, rollback, entry, rollbackErr) + } + return fmt.Errorf("restore transaction rolled back while moving original entry %s: %w", entry, err) + } + } + + installed := make([]string, 0, len(stagedEntries)) + for _, entry := range stagedEntries { + name := entry.Name() + if err := rename(filepath.Join(stage, name), filepath.Join(root, name)); err != nil { + if rollbackErr := restoreOriginal(installed); rollbackErr != nil { + keepRecovery = true + return fmt.Errorf("%s recovery files retained at %s; failed to restore original runtime after staging %s: %w", restoreRootUnsafePrefix, rollback, name, rollbackErr) + } + return fmt.Errorf("restore transaction rolled back while staging %s: %w", name, err) } + installed = append(installed, name) } return nil } -func copyDataUpdate(srcData string, dstData string, skipData map[string]bool) error { - return filepath.WalkDir(srcData, func(srcPath string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(srcData, srcPath) - if err != nil { - return err - } - if rel == "." { - return os.MkdirAll(dstData, 0755) +func collectSkipUpdatePaths(out map[string]bool, dataDir string, parent string, chunks []*domain.Chunks) error { + for _, chunk := range chunks { + if chunk == nil { + continue } - rel = filepath.ToSlash(rel) - if shouldSkipWorkerUpdate(rel, skipData) { - if entry.IsDir() { - return filepath.SkipDir + paths := []string{filepath.ToSlash(filepath.Clean(parent))} + if chunk.Path != "." { + var err error + paths, err = utils.ExpandChunkPaths(dataDir, parent, chunk.Path) + if err != nil { + return fmt.Errorf("resolve skip_update chunk %q: %w", chunk.Name, err) } - return nil } - target := filepath.Join(dstData, filepath.FromSlash(rel)) - if entry.IsDir() { - info, err := entry.Info() - if err != nil { + for _, chunkPath := range paths { + if chunk.SkipUpdate { + out[chunkPath] = true + } + if err := collectSkipUpdatePaths(out, dataDir, chunkPath, chunk.Chunks); err != nil { return err } - return os.MkdirAll(target, info.Mode().Perm()) - } - return copyPath(srcPath, target) - }) -} - -func shouldSkipWorkerUpdate(rel string, skipData map[string]bool) bool { - rel = filepath.ToSlash(filepath.Clean(rel)) - for skip := range skipData { - if skip == "" || rel == skip || strings.HasPrefix(rel, skip+"/") { - return true } } - return false + return nil } func copyPath(src string, dst string) error { - info, err := os.Stat(src) + info, err := os.Lstat(src) if err != nil { return err } + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + return os.Symlink(target, dst) + } if info.IsDir() { return filepath.WalkDir(src, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { diff --git a/apps/druid/adapters/cli/worker_push.go b/apps/druid/adapters/cli/worker_push.go index 5e0b1cdc..6e7b7546 100644 --- a/apps/druid/adapters/cli/worker_push.go +++ b/apps/druid/adapters/cli/worker_push.go @@ -11,6 +11,7 @@ import ( var workerPushArtifact string var workerPushRoot string +var workerPushPreserveReleaseManifest bool var WorkerPushCommand = &cobra.Command{ Use: "push", @@ -29,7 +30,7 @@ var WorkerPushCommand = &cobra.Command{ } repo, tag := utils.SplitArtifact(workerPushArtifact) oci := registry.NewOciClient(loadWorkerRegistryStore()) - _, err = oci.Push(workerPushRoot, repo, tag, nil, false, &scroll.File) + _, err = oci.PushWithOptions(workerPushRoot, repo, tag, nil, false, &scroll.File, registry.TransferOptions{PreserveReleaseManifest: workerPushPreserveReleaseManifest}) return err }, } @@ -38,5 +39,6 @@ func init() { WorkerCommand.AddCommand(WorkerPushCommand) WorkerPushCommand.Flags().StringVar(&workerPushArtifact, "artifact", "", "OCI artifact to push") WorkerPushCommand.Flags().StringVar(&workerPushRoot, "root", "/scroll", "Mounted runtime root path") + WorkerPushCommand.Flags().BoolVar(&workerPushPreserveReleaseManifest, "preserve-release-manifest", false, "Include the installed release manifest.json in the backup payload") WorkerPushCommand.MarkFlagRequired("artifact") } diff --git a/apps/druid/adapters/cli/worker_release_test.go b/apps/druid/adapters/cli/worker_release_test.go new file mode 100644 index 00000000..29641eef --- /dev/null +++ b/apps/druid/adapters/cli/worker_release_test.go @@ -0,0 +1,96 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/highcard-dev/daemon/internal/core/ports" +) + +func TestInspectWorkerReadsInstalledReleaseNotRegistryReference(t *testing.T) { + root := t.TempDir() + want := "sha256:" + strings.Repeat("a", 64) + raw := "{\n \"digest\": \"" + want + "\"\n}" + mustWrite(t, filepath.Join(root, "manifest.json"), raw) + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: registry.local/owned/example\napp_version: '1'\ncommands: {}\n") + result := runWorkerPull(ports.RuntimeWorkerAction{Mode: ports.RuntimeWorkerModeInspect, MountPath: root, Artifact: "unreachable.invalid/backup:latest"}) + if result.Error != "" || result.ArtifactDigest != want { + t.Fatalf("inspection = %#v", result) + } + assertFile(t, filepath.Join(root, "manifest.json"), raw) +} + +func TestInspectWorkerRejectsMissingOrInvalidInstalledDescriptor(t *testing.T) { + for _, raw := range []string{"", "{}", `{"digest":"sha256:bad"}`} { + t.Run(raw, func(t *testing.T) { + root := t.TempDir() + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: fixture\n") + if raw != "" { + mustWrite(t, filepath.Join(root, "manifest.json"), raw) + } + if result := inspectInstalledRelease(root); result.Error == "" { + t.Fatal("invalid installed release was accepted") + } + }) + } +} + +func TestPullWorkerReportsMaterializedRelease(t *testing.T) { + for _, mode := range []ports.RuntimeWorkerMode{ports.RuntimeWorkerModeCreate, ports.RuntimeWorkerModeUpdate} { + t.Run(string(mode), func(t *testing.T) { + source, root := t.TempDir(), t.TempDir() + want := "sha256:" + strings.Repeat("a", 64) + yaml := "name: registry.local/release\napp_version: '2'\ncommands: {}\n" + mustWrite(t, filepath.Join(source, "manifest.json"), `{"digest":"`+want+`"}`) + mustWrite(t, filepath.Join(source, "scroll.yaml"), yaml) + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: previous\n") + result := runWorkerPull(ports.RuntimeWorkerAction{Mode: mode, MountPath: root, Artifact: source}) + if result.Error != "" || result.ArtifactDigest != want || result.ScrollYAML != yaml { + t.Fatalf("materialized result = %#v", result) + } + }) + } +} + +func TestPullWorkerAllowsLocalScrollWithoutReleaseDescriptor(t *testing.T) { + source, root := t.TempDir(), t.TempDir() + mustWrite(t, filepath.Join(source, "scroll.yaml"), "name: local\n") + result := runWorkerPull(ports.RuntimeWorkerAction{Mode: ports.RuntimeWorkerModeCreate, MountPath: root, Artifact: source}) + if result.Error != "" || result.ArtifactDigest != "" || result.ScrollYAML != "name: local\n" { + t.Fatalf("local result = %#v", result) + } +} + +func TestPreserveSkippedUpdateDataRejectsTraversal(t *testing.T) { + root := t.TempDir() + stage, err := os.MkdirTemp(root, ".stage-") + if err != nil { + t.Fatal(err) + } + if err := preserveSkippedUpdateData(root, stage, map[string]bool{"../../outside": true}); err == nil { + t.Fatal("unsafe skip_update path accepted") + } +} + +func TestPreserveSkippedUpdateDataDoesNotFollowSymlinks(t *testing.T) { + root, stage, outside := t.TempDir(), t.TempDir(), t.TempDir() + mustWrite(t, filepath.Join(outside, "save"), "outside runtime") + if err := os.MkdirAll(filepath.Join(root, "data"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "data", "link")); err != nil { + t.Fatal(err) + } + if err := preserveSkippedUpdateData(root, stage, map[string]bool{"link/save": true}); err == nil { + t.Fatal("followed a protected path through an external symlink") + } + if err := preserveSkippedUpdateData(root, stage, map[string]bool{"link": true}); err != nil { + t.Fatal(err) + } + if target, err := os.Readlink(filepath.Join(stage, "data", "link")); err != nil || target != outside { + t.Fatalf("protected symlink was dereferenced: %q (%v)", target, err) + } + assertFile(t, filepath.Join(outside, "save"), "outside runtime") +} diff --git a/apps/druid/adapters/cli/worker_test.go b/apps/druid/adapters/cli/worker_test.go index 80a3cb27..64228298 100644 --- a/apps/druid/adapters/cli/worker_test.go +++ b/apps/druid/adapters/cli/worker_test.go @@ -1,17 +1,19 @@ package cli import ( + "errors" "net/http" "net/http/httptest" "os" "path/filepath" + "reflect" + "strings" "testing" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" - v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/highcard-dev/daemon/internal/core/services/registry" "github.com/spf13/cobra" - "oras.land/oras-go/v2/registry/remote" ) func TestWorkerPullCommandUsesRuntimeIDOnly(t *testing.T) { @@ -33,6 +35,35 @@ func TestWorkerPullCommandRequiresRuntimeID(t *testing.T) { } } +func TestWorkerPullCommandAcceptsLegacyManifestFlagWithoutChangingAction(t *testing.T) { + flag := WorkerPullCommand.Flags().Lookup("preserve-release-manifest") + if flag == nil || !flag.Hidden { + t.Fatal("worker pull should accept a hidden legacy --preserve-release-manifest flag") + } + previousValue, previousChanged := flag.Value.String(), flag.Changed + t.Cleanup(func() { + if err := flag.Value.Set(previousValue); err != nil { + t.Fatal(err) + } + flag.Changed = previousChanged + }) + before := workerPullAction + for _, arg := range []string{"--preserve-release-manifest", "--preserve-release-manifest=false"} { + if err := WorkerPullCommand.ParseFlags([]string{arg}); err != nil { + t.Fatalf("legacy flag should still be accepted: %v", err) + } + if !reflect.DeepEqual(workerPullAction, before) { + t.Fatal("legacy flag must not change worker action") + } + } +} + +func TestWorkerPushCommandSupportsReleaseManifestPreservation(t *testing.T) { + if flag := WorkerPushCommand.Flags().Lookup("preserve-release-manifest"); flag == nil { + t.Fatal("worker push should expose --preserve-release-manifest") + } +} + func TestReportWorkerResultUsesTokenOnlyWhenProvided(t *testing.T) { tests := []struct { name string @@ -70,29 +101,79 @@ func TestReportWorkerResultUsesTokenOnlyWhenProvided(t *testing.T) { } } -func TestWorkerUpdateMergePreservesSkipUpdateAndExtraFiles(t *testing.T) { - src := t.TempDir() - dst := t.TempDir() - mustWrite(t, filepath.Join(src, "scroll.yaml"), "name: next\n") - mustWrite(t, filepath.Join(src, "data", "keep", "state.txt"), "new") - mustWrite(t, filepath.Join(src, "data", "overwrite.txt"), "new") - mustWrite(t, filepath.Join(dst, "scroll.yaml"), "name: old\n") - mustWrite(t, filepath.Join(dst, "data", "keep", "state.txt"), "old") - mustWrite(t, filepath.Join(dst, "data", "overwrite.txt"), "old") - mustWrite(t, filepath.Join(dst, "data", "extra.txt"), "extra") +func TestStagedUpdateReplacesUnprotectedAndPreservesSkipUpdate(t *testing.T) { + root := t.TempDir() + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: old\n") + mustWrite(t, filepath.Join(root, "data", "keep", "state.txt"), "old") + mustWrite(t, filepath.Join(root, "data", "extra.txt"), "old") + stage, err := os.MkdirTemp(root, ".druid-worker-update-stage-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(stage) + mustWrite(t, filepath.Join(stage, "scroll.yaml"), "name: new\n") + mustWrite(t, filepath.Join(stage, "data", "keep", "state.txt"), "new") + mustWrite(t, filepath.Join(stage, "data", "replace.txt"), "new") + + if err := preserveSkippedUpdateData(root, stage, map[string]bool{"keep": true}); err != nil { + t.Fatal(err) + } + if err := replaceRestoredRoot(root, stage); err != nil { + t.Fatal(err) + } + + assertFile(t, filepath.Join(root, "scroll.yaml"), "name: new\n") + assertFile(t, filepath.Join(root, "data", "keep", "state.txt"), "old") + assertFile(t, filepath.Join(root, "data", "replace.txt"), "new") + if _, err := os.Stat(filepath.Join(root, "data", "extra.txt")); !os.IsNotExist(err) { + t.Fatalf("unprotected destination-only file should be removed, stat err = %v", err) + } +} + +func TestPreserveSkippedUpdateDataKeepsMissingPathsMissing(t *testing.T) { + root := t.TempDir() + stage, err := os.MkdirTemp(root, ".druid-worker-update-stage-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(stage) + mustWrite(t, filepath.Join(stage, "data", "keep", "state.txt"), "candidate") - if err := mergePulledRoot(src, dst, map[string]bool{"keep": true}); err != nil { + if err := preserveSkippedUpdateData(root, stage, map[string]bool{"keep": true}); err != nil { t.Fatal(err) } - assertFile(t, filepath.Join(dst, "scroll.yaml"), "name: next\n") - assertFile(t, filepath.Join(dst, "data", "keep", "state.txt"), "old") - assertFile(t, filepath.Join(dst, "data", "overwrite.txt"), "new") - assertFile(t, filepath.Join(dst, "data", "extra.txt"), "extra") + if _, err := os.Stat(filepath.Join(stage, "data", "keep")); !os.IsNotExist(err) { + t.Fatalf("missing installed skip_update path should remain missing, stat err = %v", err) + } +} + +func TestWorkerUpdatePreservesRemovedProtectedChunk(t *testing.T) { + for _, chunks := range []string{ + " - name: world\n path: world\n skip_update: true\n", + " - name: server\n path: .\n chunks:\n - name: world\n path: world\n skip_update: true\n", + } { + t.Run(chunks, func(t *testing.T) { + root := t.TempDir() + candidate := t.TempDir() + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: example\nchunks:\n"+chunks) + mustWrite(t, filepath.Join(root, "data", "world", "save.dat"), "user world") + mustWrite(t, filepath.Join(root, "data", "obsolete.txt"), "old release") + mustWrite(t, filepath.Join(candidate, "scroll.yaml"), "name: example\n") + if err := pullWorkerUpdate(root, candidate, nil, nil); err != nil { + t.Fatal(err) + } + assertFile(t, filepath.Join(root, "data", "world", "save.dat"), "user world") + if _, err := os.Stat(filepath.Join(root, "data", "obsolete.txt")); !os.IsNotExist(err) { + t.Fatalf("unprotected obsolete file retained: %v", err) + } + }) + } } func TestWorkerRestoreStagesBeforeReplacingRoot(t *testing.T) { root := t.TempDir() mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: old\n") + mustWrite(t, filepath.Join(root, "manifest.json"), `{"digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}`) mustWrite(t, filepath.Join(root, "data", "logs", "latest.log"), "old") mustWrite(t, filepath.Join(root, "data", "old-only.txt"), "old") @@ -102,12 +183,111 @@ func TestWorkerRestoreStagesBeforeReplacingRoot(t *testing.T) { } assertFile(t, filepath.Join(root, "scroll.yaml"), "name: restored\n") + assertFile(t, filepath.Join(root, "manifest.json"), `{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}`) + result := readInstalledRelease(root, true) + if result.Error != "" || result.ArtifactDigest != "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" || result.ScrollYAML != "name: restored\n" { + t.Fatalf("restored identity must describe the preserved release: %#v", result) + } assertFile(t, filepath.Join(root, "data", "logs", "latest.log"), "restored") if _, err := os.Stat(filepath.Join(root, "data", "old-only.txt")); !os.IsNotExist(err) { t.Fatalf("old root contents should be removed before restore, stat err = %v", err) } } +func TestWorkerRestorePreservesRootWhenPullFails(t *testing.T) { + root := t.TempDir() + original := map[string]string{ + "scroll.yaml": "name: original\n", + "manifest.json": `{"digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}`, + "data/logs/latest.log": "original log", + "data/old-only.txt": "original data", + } + for path, content := range original { + mustWrite(t, filepath.Join(root, path), content) + } + wantErr := errors.New("injected pull failure") + err := pullWorkerRestore(root, "registry.local/backup:1", fakeRestoreOCI{t: t, err: wantErr}) + if !errors.Is(err, wantErr) { + t.Fatalf("restore error = %v, want %v", err, wantErr) + } + for path, content := range original { + assertFile(t, filepath.Join(root, path), content) + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 3 { + t.Fatalf("restore left unexpected root entries: %v", entries) + } +} + +func TestReplaceRestoredRootRollsBackAfterStagingBegins(t *testing.T) { + root := t.TempDir() + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: original\n") + mustWrite(t, filepath.Join(root, "data", "world.txt"), "original") + stage, err := os.MkdirTemp(root, ".druid-worker-restore-stage-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(stage) + mustWrite(t, filepath.Join(stage, "a-new.txt"), "new") + mustWrite(t, filepath.Join(stage, "b-new.txt"), "new") + + stagedMoves := 0 + err = replaceRestoredRootWithRename(root, stage, func(from string, to string) error { + if strings.HasPrefix(from, stage+string(os.PathSeparator)) { + stagedMoves++ + if stagedMoves == 2 { + return errors.New("injected staging failure") + } + } + return os.Rename(from, to) + }) + if err == nil || !strings.Contains(err.Error(), "restore transaction rolled back") { + t.Fatalf("restore error = %v, want successful rollback error", err) + } + assertFile(t, filepath.Join(root, "scroll.yaml"), "name: original\n") + assertFile(t, filepath.Join(root, "data", "world.txt"), "original") + if _, err := os.Stat(filepath.Join(root, "a-new.txt")); !os.IsNotExist(err) { + t.Fatalf("staged file should be removed during rollback, stat err = %v", err) + } +} + +func TestReplaceRestoredRootMarksFailedRollbackUnsafe(t *testing.T) { + root := t.TempDir() + mustWrite(t, filepath.Join(root, "original.txt"), "original") + stage, err := os.MkdirTemp(root, ".druid-worker-restore-stage-*") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(stage) + mustWrite(t, filepath.Join(stage, "a-new.txt"), "new") + mustWrite(t, filepath.Join(stage, "b-new.txt"), "new") + + stagedMoves := 0 + err = replaceRestoredRootWithRename(root, stage, func(from string, to string) error { + if strings.HasPrefix(from, stage+string(os.PathSeparator)) { + stagedMoves++ + if stagedMoves == 2 { + return errors.New("injected staging failure") + } + } + if strings.Contains(filepath.Base(filepath.Dir(from)), "restore-rollback-") { + return errors.New("injected rollback failure") + } + return os.Rename(from, to) + }) + if err == nil || !strings.HasPrefix(err.Error(), restoreRootUnsafePrefix) { + t.Fatalf("restore error = %v, want unsafe rollback marker", err) + } + recovery, err := filepath.Glob(filepath.Join(root, ".druid-worker-restore-rollback-*", "original.txt")) + if err != nil || len(recovery) != 1 { + t.Fatalf("original data must survive failed rollback: files=%v err=%v", recovery, err) + } + assertFile(t, recovery[0], "original") +} + func TestWorkerCollectSkipUpdatePaths(t *testing.T) { root := filepath.Join(t.TempDir(), "root") mustWrite(t, filepath.Join(root, "scroll.yaml"), `name: skip-test @@ -148,7 +328,9 @@ func runWorkerPullForSkipPathTest(t *testing.T, root string) map[string]bool { t.Fatal(err) } result := map[string]bool{} - collectSkipUpdatePaths(result, "", scroll.Chunks) + if err := collectSkipUpdatePaths(result, filepath.Join(root, domain.RuntimeDataDir), "", scroll.Chunks); err != nil { + t.Fatal(err) + } return result } @@ -174,43 +356,16 @@ func assertFile(t *testing.T, path string, want string) { } type fakeRestoreOCI struct { - t *testing.T -} - -func (fakeRestoreOCI) GetRepo(string) (*remote.Repository, error) { - return nil, nil + t *testing.T + err error } -func (fakeRestoreOCI) Push(string, string, string, map[string]string, bool, *domain.File) (v1.Descriptor, error) { - return v1.Descriptor{}, nil -} - -func (f fakeRestoreOCI) Pull(dir string, artifact string) error { - return f.PullSelective(dir, artifact, true, nil) -} - -func (f fakeRestoreOCI) PullSelective(dir string, artifact string, includeData bool, progress *domain.SnapshotProgress) error { +func (f fakeRestoreOCI) PullSelectiveWithOptions(dir string, artifact string, includeData bool, progress *domain.SnapshotProgress, options registry.TransferOptions) error { + if !options.PreserveReleaseManifest { + f.t.Fatal("restore must preserve the release manifest") + } mustWrite(f.t, filepath.Join(dir, "scroll.yaml"), "name: restored\n") + mustWrite(f.t, filepath.Join(dir, "manifest.json"), `{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}`) mustWrite(f.t, filepath.Join(dir, "data", "logs", "latest.log"), "restored") - return nil -} - -func (fakeRestoreOCI) FetchFile(string, string) ([]byte, error) { - return nil, os.ErrNotExist -} - -func (fakeRestoreOCI) ValidateCredentials(string, string, string) error { - return nil -} - -func (fakeRestoreOCI) ResolveDigest(string) (string, error) { - return "sha256:restored", nil -} - -func (fakeRestoreOCI) ResolveAnnotationInfo(string) (domain.AnnotationInfo, error) { - return domain.AnnotationInfo{}, nil -} - -func (fakeRestoreOCI) CanUpdateTag(v1.Descriptor, string, string) (bool, error) { - return false, nil + return f.err } diff --git a/apps/druid/adapters/daemonclient/openapi_client.go b/apps/druid/adapters/daemonclient/openapi_client.go index 0f4179c0..88d1de2d 100644 --- a/apps/druid/adapters/daemonclient/openapi_client.go +++ b/apps/druid/adapters/daemonclient/openapi_client.go @@ -80,10 +80,7 @@ func (c *OpenAPIClient) CreateScroll(ctx context.Context, name string, artifact } func (c *OpenAPIClient) UpdateScroll(ctx context.Context, id string, artifact string, registryCredentials []api.RegistryCredential) (*api.RuntimeScroll, error) { - request := api.UpdateScrollJSONRequestBody{} - if artifact != "" { - request.Artifact = &artifact - } + request := api.UpdateScrollJSONRequestBody{Artifact: artifact} if len(registryCredentials) > 0 { request.RegistryCredentials = ®istryCredentials } diff --git a/apps/druid/adapters/http/handlers/routes.go b/apps/druid/adapters/http/handlers/routes.go index 4e4a0724..adb9fc97 100644 --- a/apps/druid/adapters/http/handlers/routes.go +++ b/apps/druid/adapters/http/handlers/routes.go @@ -63,6 +63,8 @@ func RegisterPublicRoutes(app *fiber.App, handlers RouteHandlers) { app.Get("/:id/api/v1/health", handlers.Server.GetHealthAuth) app.Get("/:id/api/v1/token", handlers.Server.CreateDaemonToken) app.Get("/:id/api/v1/scroll", handlers.Server.GetDaemonScroll) + app.Get("/:id/api/v1/release", func(c *fiber.Ctx) error { return handlers.Server.GetInstalledRelease(c, c.Params("id")) }) + app.Post("/:id/api/v1/update", func(c *fiber.Ctx) error { return handlers.Server.UpdateScroll(c, c.Params("id")) }) app.Put("/:id/api/v1/scroll/commands/:command", handlers.Server.AddDaemonCommand) app.Delete("/:id/api/v1/scroll/commands/:command", handlers.Server.RemoveDaemonCommand) app.Post("/:id/api/v1/command", handlers.Server.RunDaemonCommand) diff --git a/apps/druid/adapters/http/handlers/routes_test.go b/apps/druid/adapters/http/handlers/routes_test.go index a66a273e..11fca1de 100644 --- a/apps/druid/adapters/http/handlers/routes_test.go +++ b/apps/druid/adapters/http/handlers/routes_test.go @@ -3,9 +3,13 @@ package handlers import ( "net/http" "net/http/httptest" + "strings" "testing" "github.com/gofiber/fiber/v2" + appservices "github.com/highcard-dev/daemon/apps/druid/core/services" + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" ) func TestRouteSplitKeepsManagementAndPublicSurfacesSeparate(t *testing.T) { @@ -68,3 +72,63 @@ func requestStatus(t *testing.T, app *fiber.App, path string) int { defer resp.Body.Close() return resp.StatusCode } + +type ownerRouteStore struct{ ports.RuntimeScrollStore } + +func (ownerRouteStore) GetScroll(id string) (*domain.RuntimeScroll, error) { + return &domain.RuntimeScroll{ID: id, OwnerID: "owner"}, nil +} + +type ownerRouteAuth struct { + ports.AuthorizerServiceInterface +} + +func (ownerRouteAuth) CheckHeader(c *fiber.Ctx) (*ports.AuthContext, error) { + if c.Get("Authorization") == "" { + return nil, nil + } + return &ports.AuthContext{Subject: strings.TrimPrefix(c.Get("Authorization"), "Bearer ")}, nil +} + +type ownerRouteBackend struct{ ports.RuntimeBackendInterface } + +func TestPublicReleaseRoutesEnforceOwnerBeforeReadingOrUpdating(t *testing.T) { + supervisor, err := appservices.NewRuntimeSupervisor(ownerRouteStore{}, nil, ports.RuntimeBackendFactoryFunc(func(ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return ownerRouteBackend{}, nil + })) + if err != nil { + t.Fatal(err) + } + app := fiber.New() + RegisterPublicRoutes(app, RouteHandlers{Server: NewRuntimeServer(NewHealthHandler(), NewScrollHandler(supervisor, ownerRouteAuth{})), Websocket: &WebsocketHandler{}}) + for _, route := range []struct{ method, path string }{{http.MethodGet, "/owned/api/v1/release"}, {http.MethodPost, "/owned/api/v1/update"}} { + for _, auth := range []struct { + header string + status int + }{{"", http.StatusUnauthorized}, {"Bearer other-owner", http.StatusForbidden}} { + req := httptest.NewRequest(route.method, route.path, strings.NewReader(`{"artifact":"registry/scroll:mutable"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", auth.header) + res, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != auth.status { + t.Fatalf("%s %s: got %d, want %d", route.method, route.path, res.StatusCode, auth.status) + } + } + } + // An owner reaches the explicit-update validator, not a 404 or another API. + req := httptest.NewRequest(http.MethodPost, "/owned/api/v1/update", strings.NewReader(`{"artifact":"registry/scroll:mutable"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer owner") + res, err := app.Test(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusBadRequest { + t.Fatalf("owner mutable update status = %d, want 400", res.StatusCode) + } +} diff --git a/apps/druid/adapters/http/handlers/scroll_handler.go b/apps/druid/adapters/http/handlers/scroll_handler.go index 68fc4cc8..3b5e9063 100644 --- a/apps/druid/adapters/http/handlers/scroll_handler.go +++ b/apps/druid/adapters/http/handlers/scroll_handler.go @@ -124,6 +124,17 @@ func (h *ScrollHandler) GetScroll(c *fiber.Ctx, id string) error { return c.JSON(runtimeScroll) } +func (h *ScrollHandler) GetInstalledRelease(c *fiber.Ctx, id string) error { + if _, err := h.getScroll(id); err != nil { + return err + } + release, err := h.supervisor.InstalledRelease(c.UserContext(), id) + if err != nil { + return err + } + return c.JSON(release) +} + func (h *ScrollHandler) DeleteScroll(c *fiber.Ctx, id string) error { runtimeScroll, err := h.getScroll(id) if err != nil { @@ -170,12 +181,14 @@ func (h *ScrollHandler) UpdateScroll(c *fiber.Ctx, id string) error { return fiber.NewError(fiber.StatusBadRequest, err.Error()) } } - artifact := "" - if request.Artifact != nil { - artifact = *request.Artifact - } - runtimeScroll, err := h.supervisor.Update(id, artifact, registryCredentials(request.RegistryCredentials)) + runtimeScroll, err := h.supervisor.Update(id, request.Artifact, registryCredentials(request.RegistryCredentials), request.ExpectedInstalledDigest) if err != nil { + if errors.Is(err, appservices.ErrInstalledReleaseChanged) { + return fiber.NewError(fiber.StatusConflict, err.Error()) + } + if errors.Is(err, appservices.ErrUnacceptedUpdate) { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } return err } return c.JSON(runtimeScroll) diff --git a/apps/druid/core/services/runtime_access.go b/apps/druid/core/services/runtime_access.go index dd2e0b13..ff2538be 100644 --- a/apps/druid/core/services/runtime_access.go +++ b/apps/druid/core/services/runtime_access.go @@ -5,6 +5,7 @@ import ( "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" ) func (s *RuntimeSupervisor) Run(id string, command string) (*domain.RuntimeScroll, error) { @@ -12,20 +13,36 @@ func (s *RuntimeSupervisor) Run(id string, command string) (*domain.RuntimeScrol } func (s *RuntimeSupervisor) RunWithContext(ctx context.Context, id string, command string) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) session, err := s.sessionFor(id) if err != nil { + unlock() return nil, err } - return session.RunWithContext(ctx, command) + session.Start() + longRunning, err := session.beginRun(command) + unlock() + if err != nil { + return nil, err + } + return session.finishRun(ctx, longRunning) } // RunAndWait waits for the requested command rather than every command in the runtime queue. func (s *RuntimeSupervisor) RunAndWait(id string, command string) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) session, err := s.sessionFor(id) if err != nil { + unlock() return nil, err } - if err := session.AddTempItemWithWait(command); err != nil { + session.Start() + wait, err := session.enqueueQueueItem(command, coreservices.AddItemOptions{Wait: true}, nil) + unlock() + if err != nil { + return nil, err + } + if err := wait(); err != nil { return nil, err } return s.store.GetScroll(id) @@ -56,18 +73,30 @@ func (s *RuntimeSupervisor) ApplyRouting(id string, assignments []domain.Runtime } func (s *RuntimeSupervisor) Backup(id string, artifact string, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) + defer unlock() session, err := s.sessionFor(id) if err != nil { return nil, err } + wasRunning := sessionWasRunning(session) + if err := session.StopRuntimeForMaintenance(); err != nil { + session.markError(err) + return nil, err + } if err := session.Backup(context.Background(), artifact, registryCredentials); err != nil { session.markError(err) return nil, err } + if wasRunning { + return s.startScroll(id) + } return s.store.GetScroll(id) } func (s *RuntimeSupervisor) Restore(id string, artifact string, restart bool, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) + defer unlock() session, err := s.sessionFor(id) if err != nil { return nil, err @@ -75,10 +104,17 @@ func (s *RuntimeSupervisor) Restore(id string, artifact string, restart bool, re session.mu.Lock() root := session.runtimeScroll.Root session.mu.Unlock() - if err := session.StopRuntime(); err != nil { + if err := session.StopRuntimeForMaintenance(); err != nil { session.markError(err) return nil, err } + session.mu.Lock() + session.runtimeScroll.InstalledReleaseState = domain.InstalledReleasePending + err = s.store.UpdateScroll(session.runtimeScroll) + session.mu.Unlock() + if err != nil { + return nil, err + } materialized, err := s.runPullWorker(context.Background(), s.runtimeBackend, ports.RuntimeWorkerModeRestore, id, artifact, root, registryCredentials, "") if err != nil { session.markError(err) @@ -89,11 +125,17 @@ func (s *RuntimeSupervisor) Restore(id string, artifact string, restart bool, re return nil, err } if restart { - return s.StartScroll(id) + return s.startScroll(id) } return s.store.GetScroll(id) } +func sessionWasRunning(session *RuntimeSession) bool { + session.mu.Lock() + defer session.mu.Unlock() + return session.runtimeScroll.Status == domain.RuntimeScrollStatusRunning +} + func (s *RuntimeSupervisor) ScrollFile(id string) (*domain.File, error) { session, err := s.sessionFor(id) if err != nil { diff --git a/apps/druid/core/services/runtime_backup_failure_test.go b/apps/druid/core/services/runtime_backup_failure_test.go new file mode 100644 index 00000000..c7c1141d --- /dev/null +++ b/apps/druid/core/services/runtime_backup_failure_test.go @@ -0,0 +1,36 @@ +package services + +import ( + "errors" + "strings" + "testing" + + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +func TestBackupUnknownFailureDoesNotRestart(t *testing.T) { + store := newTestStateStore(t) + runtime := &domain.RuntimeScroll{ + ID: "backup-unknown", Artifact: "registry.local/original:1", Root: "runtime://backup-unknown", + ScrollName: "original", ScrollYAML: strings.Replace(cachedScrollYAML("start"), "run: once", "run: persistent", 1), + Status: domain.RuntimeScrollStatusRunning, Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtime); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{backupErr: errors.New("backup worker connection lost"), + procedureStatusUpdates: []ports.ProcedureStatusUpdate{{Procedure: "start.0", Status: domain.ScrollLockStatusRunning}}} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + if _, err := supervisor.Backup(runtime.ID, "registry.local/backups:1", nil); err == nil { + t.Fatal("expected backup error") + } + stored, err := store.GetScroll(runtime.ID) + if err != nil { + t.Fatal(err) + } + if stored.Status != domain.RuntimeScrollStatusError || stored.LastError == "" { + t.Fatalf("unconfirmed backup termination must remain stopped with an error; status=%s error=%q", stored.Status, stored.LastError) + } +} diff --git a/apps/druid/core/services/runtime_command_maintenance_test.go b/apps/druid/core/services/runtime_command_maintenance_test.go new file mode 100644 index 00000000..de72aa7e --- /dev/null +++ b/apps/druid/core/services/runtime_command_maintenance_test.go @@ -0,0 +1,94 @@ +package services + +import ( + "testing" + "time" + + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +func TestCommandAdmissionWaitsForMaintenance(t *testing.T) { + for _, sync := range []bool{false, true} { + t.Run(map[bool]string{false: "run", true: "run-and-wait"}[sync], func(t *testing.T) { + store := newTestStateStore(t) + fixture := &domain.RuntimeScroll{ID: "command-maintenance", Root: "runtime://fixture", ScrollYAML: cachedScrollYAML("start"), Status: domain.RuntimeScrollStatusStopped} + if err := store.CreateScroll(fixture); err != nil { + t.Fatal(err) + } + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{}) + unlock := supervisor.lockRuntimeOperation(fixture.ID) + defer func() { + if unlock != nil { + unlock() + } + }() + done := make(chan error, 1) + go func() { + var err error + if sync { + _, err = supervisor.RunAndWait(fixture.ID, "missing") + } else { + _, err = supervisor.Run(fixture.ID, "missing") + } + done <- err + }() + select { + case err := <-done: + t.Fatalf("command reached session during maintenance: %v", err) + case <-time.After(50 * time.Millisecond): + } + unlock() + unlock = nil + select { + case err := <-done: + if err == nil { + t.Fatal("missing command accepted") + } + case <-time.After(time.Second): + t.Fatal("command admission did not resume") + } + }) + } +} + +func TestCommandWaitDoesNotHoldMaintenanceLock(t *testing.T) { + store := newTestStateStore(t) + fixture := &domain.RuntimeScroll{ID: "command-wait", Root: "runtime://fixture", ScrollYAML: cachedScrollYAML("start"), Status: domain.RuntimeScrollStatusStopped} + if err := store.CreateScroll(fixture); err != nil { + t.Fatal(err) + } + started, release := make(chan struct{}), make(chan struct{}) + backend := &fakeWorkerBackend{runCommand: func(ports.RuntimeCommand) (*int, error) { + close(started) + <-release + code := 0 + return &code, nil + }} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + done := make(chan error, 1) + go func() { _, err := supervisor.RunAndWait(fixture.ID, "start"); done <- err }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("command did not start") + } + acquired := make(chan struct{}) + go func() { unlock := supervisor.lockRuntimeOperation(fixture.ID); close(acquired); unlock() }() + select { + case <-acquired: + case <-time.After(time.Second): + close(release) + t.Fatal("command wait blocks maintenance admission") + } + close(release) + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("command waiter did not complete") + } +} diff --git a/apps/druid/core/services/runtime_lifecycle.go b/apps/druid/core/services/runtime_lifecycle.go index a0df0ef3..e6870c30 100644 --- a/apps/druid/core/services/runtime_lifecycle.go +++ b/apps/druid/core/services/runtime_lifecycle.go @@ -7,6 +7,8 @@ func (s *RuntimeSupervisor) Delete(id string) error { } func (s *RuntimeSupervisor) DeleteWithPolicy(id string, purgeData bool) error { + unlock := s.lockRuntimeOperation(id) + defer unlock() s.mu.Lock() session := s.sessions[id] delete(s.sessions, id) @@ -28,10 +30,17 @@ func (s *RuntimeSupervisor) DeleteWithPolicy(id string, purgeData bool) error { } func (s *RuntimeSupervisor) StartScroll(id string) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) + defer unlock() + return s.startScroll(id) +} + +func (s *RuntimeSupervisor) startScroll(id string) (*domain.RuntimeScroll, error) { session, err := s.sessionFor(id) if err != nil { return nil, err } + session.Start() if err := session.AutoStartServe(); err != nil { session.markError(err) return nil, err @@ -52,6 +61,8 @@ func (s *RuntimeSupervisor) StartScroll(id string) (*domain.RuntimeScroll, error } func (s *RuntimeSupervisor) Stop(id string) (*domain.RuntimeScroll, error) { + unlock := s.lockRuntimeOperation(id) + defer unlock() session, err := s.detachSession(id) if err != nil { return nil, err diff --git a/apps/druid/core/services/runtime_materialization.go b/apps/druid/core/services/runtime_materialization.go index 9db9ce7f..3349f9d9 100644 --- a/apps/druid/core/services/runtime_materialization.go +++ b/apps/druid/core/services/runtime_materialization.go @@ -87,19 +87,3 @@ func resolveArtifactMinDisk(artifact string, registryCredentials []domain.Regist } return info.MinDisk } - -func resolveArtifactDigest(artifact string, registryCredentials []domain.RegistryCredential) string { - if artifact == "" { - return "" - } - if _, err := os.Stat(artifact); err == nil { - return "" - } - oci := registry.NewOciClient(registry.NewCredentialStore(registryCredentials)) - digest, err := oci.ResolveDigest(artifact) - if err != nil { - logger.Log().Warn("Unable to resolve artifact digest", zap.String("artifact", artifact), zap.Error(err)) - return "" - } - return digest -} diff --git a/apps/druid/core/services/runtime_release.go b/apps/druid/core/services/runtime_release.go new file mode 100644 index 00000000..dd957c42 --- /dev/null +++ b/apps/druid/core/services/runtime_release.go @@ -0,0 +1,82 @@ +package services + +import ( + "context" + "fmt" + + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +// InstalledRelease serves the descriptor reported by the last successful pull. +// The operation lock serializes queries with update/restore transitions. +func (s *RuntimeSupervisor) InstalledRelease(ctx context.Context, id string) (map[string]string, error) { + unlock := s.lockRuntimeOperation(id) + defer unlock() + runtime, err := s.store.GetScroll(id) + if err != nil { + return nil, err + } + return s.installedRelease(ctx, runtime) +} + +func (s *RuntimeSupervisor) installedRelease(ctx context.Context, runtime *domain.RuntimeScroll) (map[string]string, error) { + switch runtime.InstalledReleaseState { + case domain.InstalledReleaseReady: + return installedReleaseIdentity(runtime.Root, runtime.ScrollYAML, runtime.ArtifactDigest) + case "": // Older records need one read of the actual installed descriptor. + default: + // A lost callback may leave a worker still replacing files. Do not + // inspect concurrently or advertise the previous cached release. + return nil, fmt.Errorf("installed release is unconfirmed after an incomplete materialization") + } + installed, err := s.runPullWorker(ctx, s.runtimeBackend, ports.RuntimeWorkerModeInspect, runtime.ID, runtime.Artifact, runtime.Root, nil, "") + if err != nil { + return nil, err + } + identity, err := installedReleaseIdentity(runtime.Root, string(installed.ScrollYAML), installed.ArtifactDigest) + if err != nil { + return nil, err + } + // Keep the live session in sync so subsequent status writes cannot replace + // the newly confirmed identity with its old copy. + s.mu.Lock() + session := s.sessions[runtime.ID] + s.mu.Unlock() + current := runtime + if session != nil { + session.mu.Lock() + defer session.mu.Unlock() + current = session.runtimeScroll + } + updated := *current + updated.ArtifactDigest = installed.ArtifactDigest + updated.ScrollYAML = string(installed.ScrollYAML) + updated.InstalledReleaseState = domain.InstalledReleaseReady + scrollService, err := coreservices.NewCachedScrollServiceWithPorts(updated.Root, installed.ScrollYAML, updated.ReservedPorts) + if err != nil { + return nil, err + } + updated.ScrollName = scrollService.GetCurrent().Name + if err := s.store.UpdateScroll(&updated); err != nil { + return nil, err + } + *runtime = updated + if session != nil { + *session.runtimeScroll = updated + session.scrollService = scrollService + } + return identity, nil +} + +func installedReleaseIdentity(root, yaml, digest string) (map[string]string, error) { + scroll, err := domain.NewScrollFromBytes(root, []byte(yaml)) + if err != nil { + return nil, err + } + if !acceptedUpdateReference.MatchString(scroll.Name + "@" + digest) { + return nil, fmt.Errorf("invalid installed release identity") + } + return map[string]string{"repository": scroll.Name, "digest": digest}, nil +} diff --git a/apps/druid/core/services/runtime_release_test.go b/apps/druid/core/services/runtime_release_test.go new file mode 100644 index 00000000..0f9057dc --- /dev/null +++ b/apps/druid/core/services/runtime_release_test.go @@ -0,0 +1,175 @@ +package services + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +func TestMaterializationCachesInstalledReleaseAcrossRestart(t *testing.T) { + for _, operation := range []string{"create", "update", "restore"} { + t.Run(operation, func(t *testing.T) { + store := newTestStateStore(t) + callbacks := NewWorkerCallbackManager() + digest := "sha256:" + strings.Repeat("a", 64) + backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: releaseScrollYAML(), digest: digest} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") + id := "cached-release" + if operation != "create" { + if err := store.CreateScroll(&domain.RuntimeScroll{ID: id, Root: "runtime://cached-release", Artifact: "old", ArtifactDigest: digest, InstalledReleaseState: domain.InstalledReleaseReady, ScrollYAML: cachedScrollYAML(""), Status: domain.RuntimeScrollStatusStopped}); err != nil { + t.Fatal(err) + } + } + var result *domain.RuntimeScroll + var err error + switch operation { + case "create": + result, err = supervisor.Create(t.TempDir(), id, nil) + case "update": + result, err = supervisor.Update(id, "registry.local/release@"+digest, nil, &digest) + case "restore": + result, err = supervisor.Restore(id, "registry.local/backup:v1", false, nil) + } + if err != nil { + t.Fatal(err) + } + if result.InstalledReleaseState != domain.InstalledReleaseReady { + t.Fatalf("successful %s left release %q", operation, result.InstalledReleaseState) + } + // A new supervisor has neither callbacks nor access to the volume. + restarted := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + for i := 0; i < 2; i++ { + identity, err := restarted.InstalledRelease(context.Background(), result.ID) + if err != nil || identity["repository"] != "registry.local/release" || identity["digest"] != digest { + t.Fatalf("cached release = %v, %v", identity, err) + } + } + if backend.spawnCount != 1 { + t.Fatalf("spawned %d workers; want only the materialization worker", backend.spawnCount) + } + }) + } +} + +func TestLegacyInstalledReleaseInspectedOnceAndSessionUpdated(t *testing.T) { + store := newTestStateStore(t) + legacy := &domain.RuntimeScroll{ID: "legacy", Root: "runtime://legacy", Artifact: "registry.local/backup:v1", ArtifactDigest: "sha256:" + strings.Repeat("b", 64), ScrollYAML: cachedScrollYAML(""), Status: domain.RuntimeScrollStatusStopped} + if err := store.CreateScroll(legacy); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + digest := "sha256:" + strings.Repeat("a", 64) + backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: releaseScrollYAML(), digest: digest} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") + session, err := supervisor.sessionFor(legacy.ID) + if err != nil { + t.Fatal(err) + } + identity, err := supervisor.InstalledRelease(context.Background(), legacy.ID) + if err != nil || identity["digest"] != digest || backend.action.Mode != ports.RuntimeWorkerModeInspect { + t.Fatalf("legacy repair = %v, %v", identity, err) + } + // A normal session write must not resurrect its legacy identity. + session.markError(errors.New("unrelated command error")) + restarted := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + identity, err = restarted.InstalledRelease(context.Background(), legacy.ID) + if err != nil || identity["digest"] != digest || backend.spawnCount != 1 { + t.Fatalf("persisted repair = %v, %v; workers = %d", identity, err, backend.spawnCount) + } +} + +func TestInstalledReleaseRejectsInvalidConfirmedIdentityWithoutWorker(t *testing.T) { + for _, digest := range []string{"", "sha256:invalid"} { + t.Run(digest, func(t *testing.T) { + store := newTestStateStore(t) + if err := store.CreateScroll(&domain.RuntimeScroll{ID: "invalid", ScrollYAML: "name: local\n", ArtifactDigest: digest, InstalledReleaseState: domain.InstalledReleaseReady}); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + if _, err := supervisor.InstalledRelease(context.Background(), "invalid"); err == nil || !strings.Contains(err.Error(), "invalid installed release identity") { + t.Fatalf("invalid release query = %v", err) + } + if backend.spawnCount != 0 { + t.Fatal("invalid confirmed identity triggered inspection") + } + }) + } +} + +type failReleaseStateStore struct { + ports.RuntimeScrollStore + state domain.InstalledReleaseState +} + +func (s *failReleaseStateStore) UpdateScroll(scroll *domain.RuntimeScroll) error { + if scroll.InstalledReleaseState == s.state { + return errors.New("injected release state persistence failure") + } + return s.RuntimeScrollStore.UpdateScroll(scroll) +} + +func TestMaterializationRequiresDurableInvalidation(t *testing.T) { + for _, operation := range []string{"update", "restore", "ensure"} { + t.Run(operation, func(t *testing.T) { + store := &failReleaseStateStore{RuntimeScrollStore: newTestStateStore(t), state: domain.InstalledReleasePending} + yaml := cachedScrollYAML("") + if operation == "ensure" { + yaml = "" + } + runtime := &domain.RuntimeScroll{ID: "persist-first", Root: "runtime://persist-first", ScrollYAML: yaml, InstalledReleaseState: domain.InstalledReleaseReady, Status: domain.RuntimeScrollStatusStopped} + if err := store.CreateScroll(runtime); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(NewWorkerCallbackManager(), "http://worker-callback") + var err error + switch operation { + case "update": + _, err = supervisor.Update(runtime.ID, "registry.local/release@sha256:"+strings.Repeat("a", 64), nil, nil) + case "restore": + _, err = supervisor.Restore(runtime.ID, "registry.local/backup:v1", false, nil) + case "ensure": + _, err = supervisor.Ensure(EnsureOptions{Name: runtime.ID, Artifact: t.TempDir()}) + } + if err == nil || !strings.Contains(err.Error(), "persistence failure") || backend.spawnCount != 0 { + t.Fatalf("materialization proceeded without durable invalidation: %v, %d workers", err, backend.spawnCount) + } + }) + } +} + +func TestLegacyInspectionDoesNotConfirmCacheWhenPersistenceFails(t *testing.T) { + store := &failReleaseStateStore{RuntimeScrollStore: newTestStateStore(t), state: domain.InstalledReleaseReady} + if err := store.CreateScroll(&domain.RuntimeScroll{ID: "legacy", Artifact: "backup:v1", Root: "runtime://legacy", ScrollYAML: cachedScrollYAML("")}); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: releaseScrollYAML(), digest: "sha256:" + strings.Repeat("a", 64)} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") + session, err := supervisor.sessionFor("legacy") + if err != nil { + t.Fatal(err) + } + if _, err := supervisor.InstalledRelease(context.Background(), "legacy"); err == nil { + t.Fatal("inspection ignored persistence failure") + } + session.mu.Lock() + defer session.mu.Unlock() + if session.runtimeScroll.InstalledReleaseState != "" || session.runtimeScroll.ArtifactDigest != "" { + t.Fatal("failed write confirmed the in-memory cache") + } +} + +func releaseScrollYAML() string { + return strings.Replace(cachedScrollYAML(""), "name: cached", "name: registry.local/release", 1) +} diff --git a/apps/druid/core/services/runtime_restore_failure_test.go b/apps/druid/core/services/runtime_restore_failure_test.go new file mode 100644 index 00000000..d9333903 --- /dev/null +++ b/apps/druid/core/services/runtime_restore_failure_test.go @@ -0,0 +1,69 @@ +package services + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/highcard-dev/daemon/internal/core/domain" + "github.com/highcard-dev/daemon/internal/core/ports" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +func TestRestoreIndeterminateFailureDoesNotRestart(t *testing.T) { + for _, failure := range []string{"spawn", "worker exit", "lost callback", "error callback"} { + t.Run(failure, func(t *testing.T) { + store := newTestStateStore(t) + runtime := &domain.RuntimeScroll{ + ID: "restore-unknown", Artifact: "registry.local/original:1", Root: "runtime://restore-unknown", + ScrollName: "original", ScrollYAML: strings.Replace(cachedScrollYAML("start"), "run: once", "run: persistent", 1), + Status: domain.RuntimeScrollStatusRunning, Procedures: domain.ProcedureStatusMap{}, + InstalledReleaseState: domain.InstalledReleaseReady, + ArtifactDigest: "sha256:" + strings.Repeat("a", 64), + } + if err := store.CreateScroll(runtime); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{procedureStatusUpdates: []ports.ProcedureStatusUpdate{{Procedure: "start.0", Status: domain.ScrollLockStatusRunning}}} + switch failure { + case "spawn": + backend.workerErr = errors.New("worker creation response lost") + case "worker exit": + done := make(chan error, 1) + done <- errors.New("worker killed during root replacement") + backend.workerDone = done + case "error callback": + backend.callbacks = callbacks + backend.workerResult = &ports.RuntimeWorkerResult{Error: "worker failed without recovery classification"} + } + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") + supervisor.SetWorkerTimeout(20 * time.Millisecond) + if _, err := supervisor.Restore(runtime.ID, "registry.local/backups:1", true, nil); err == nil { + t.Fatal("expected worker failure") + } + stored, err := store.GetScroll(runtime.ID) + if err != nil { + t.Fatal(err) + } + if stored.Status != domain.RuntimeScrollStatusError || stored.LastError == "" { + t.Fatalf("unknown restore outcome must remain stopped with an error, got status=%s error=%q", stored.Status, stored.LastError) + } + if stored.InstalledReleaseState != domain.InstalledReleasePending { + t.Fatalf("uncertain release was confirmed: %s", stored.InstalledReleaseState) + } + // A daemon restart must not serve the old digest or inspect a volume + // which an orphaned worker could still be modifying. + restarted := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + if _, err := restarted.InstalledRelease(context.Background(), runtime.ID); err == nil || !strings.Contains(err.Error(), "unconfirmed") { + t.Fatalf("uncertain release query = %v", err) + } + if backend.spawnCount != 1 { + t.Fatal("release query spawned another worker after uncertain restore") + } + }) + } +} diff --git a/apps/druid/core/services/runtime_session_cache.go b/apps/druid/core/services/runtime_session_cache.go index 3fc7a79c..19145426 100644 --- a/apps/druid/core/services/runtime_session_cache.go +++ b/apps/druid/core/services/runtime_session_cache.go @@ -1,11 +1,25 @@ package services import ( + "sync" + "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/utils/logger" "go.uber.org/zap" ) +func (s *RuntimeSupervisor) lockRuntimeOperation(id string) func() { + s.operationsMu.Lock() + operation := s.operations[id] + if operation == nil { + operation = &sync.Mutex{} + s.operations[id] = operation + } + s.operationsMu.Unlock() + operation.Lock() + return operation.Unlock +} + func (s *RuntimeSupervisor) detachSession(id string) (*RuntimeSession, error) { s.mu.Lock() session := s.sessions[id] diff --git a/apps/druid/core/services/runtime_session_commands.go b/apps/druid/core/services/runtime_session_commands.go index 949d9935..0b2e8ff2 100644 --- a/apps/druid/core/services/runtime_session_commands.go +++ b/apps/druid/core/services/runtime_session_commands.go @@ -121,34 +121,42 @@ func (s *RuntimeSession) Run(command string) (*domain.RuntimeScroll, error) { } func (s *RuntimeSession) RunWithContext(ctx context.Context, command string) (*domain.RuntimeScroll, error) { + longRunning, err := s.beginRun(command) + if err != nil { + return nil, err + } + return s.finishRun(ctx, longRunning) +} + +func (s *RuntimeSession) beginRun(command string) (bool, error) { s.refreshCommandState() targetCommand, err := s.scrollService.GetCommand(command) if err != nil { s.markError(err) - return nil, err + return false, err } longRunning := targetCommand.Run == domain.RunModeRestart || targetCommand.Run == domain.RunModePersistent s.rememberDoneDependencies(targetCommand, map[string]bool{}) if err := s.AddTempItem(command); err != nil { s.markError(err) - return nil, err + return false, err } + return longRunning, nil +} + +func (s *RuntimeSession) finishRun(ctx context.Context, longRunning bool) (*domain.RuntimeScroll, error) { if !longRunning { if err := s.WaitUntilEmptyContext(ctx); err != nil { - s.markError(err) return nil, err } } s.mu.Lock() - s.runtimeScroll.Status = deriveRuntimeScrollStatus(s.runtimeScroll.Procedures, s.scrollService.GetFile().Commands) - err = s.store.UpdateScroll(s.runtimeScroll) id := s.runtimeScroll.ID s.mu.Unlock() - if err != nil { - return nil, err - } + // Queue callbacks own status updates; a waiter must not overwrite state + // after concurrent maintenance has installed a different session/release. return s.store.GetScroll(id) } diff --git a/apps/druid/core/services/runtime_session_queue.go b/apps/druid/core/services/runtime_session_queue.go index 874002bf..d1104195 100644 --- a/apps/druid/core/services/runtime_session_queue.go +++ b/apps/druid/core/services/runtime_session_queue.go @@ -57,11 +57,21 @@ func (s *RuntimeSession) addQueueItem(cmd string, options coreservices.AddItemOp } func (s *RuntimeSession) addQueueItemWithEnv(cmd string, options coreservices.AddItemOptions, procedureEnv map[string]map[string]string) error { + wait, err := s.enqueueQueueItem(cmd, options, procedureEnv) + if err != nil { + return err + } + return wait() +} + +// enqueueQueueItem separates admission from waiting. Supervisors serialize +// admission with maintenance without holding a lock while a command runs. +func (s *RuntimeSession) enqueueQueueItem(cmd string, options coreservices.AddItemOptions, procedureEnv map[string]map[string]string) (func() error, error) { logger.Log().Debug("Running command", zap.String("cmd", cmd)) command, err := s.scrollService.GetCommand(cmd) if err != nil { - return err + return nil, err } s.queueMu.Lock() @@ -71,12 +81,12 @@ func (s *RuntimeSession) addQueueItemWithEnv(cmd string, options coreservices.Ad if item != nil { if currentStatus != domain.ScrollLockStatusDone && currentStatus != domain.ScrollLockStatusError { s.queueMu.Unlock() - return coreservices.ErrAlreadyInQueue + return nil, coreservices.ErrAlreadyInQueue } } if hasCurrentStatus && currentStatus == domain.ScrollLockStatusDone && command.Run == domain.RunModeOnce && !options.Force { s.queueMu.Unlock() - return coreservices.ErrCommandDoneOnce + return nil, coreservices.ErrCommandDoneOnce } var doneChan chan struct{} @@ -90,21 +100,19 @@ func (s *RuntimeSession) addQueueItemWithEnv(cmd string, options coreservices.Ad s.triggerRunQueue() - if options.Wait { - <-doneChan - s.queueMu.Lock() - item := s.queue[cmd] - var itemErr error - if item != nil { - itemErr = item.err - } - s.queueMu.Unlock() - if itemErr != nil { - return itemErr + return func() error { + if options.Wait { + <-doneChan + s.queueMu.Lock() + itemErr := item.err + s.queueMu.Unlock() + if itemErr != nil { + return itemErr + } } - } - return nil + return nil + }, nil } func (s *RuntimeSession) HydrateFromState(statuses domain.ProcedureStatusMap) error { diff --git a/apps/druid/core/services/runtime_session_runtime.go b/apps/druid/core/services/runtime_session_runtime.go index 19747241..a834ddc9 100644 --- a/apps/druid/core/services/runtime_session_runtime.go +++ b/apps/druid/core/services/runtime_session_runtime.go @@ -77,6 +77,17 @@ func hasRunningProcedure(procedures domain.ProcedureStatusMap) bool { } func (s *RuntimeSession) StopRuntime() error { + return s.stopRuntime(true) +} + +// StopRuntimeForMaintenance keeps the runtime quiescent after the backend has +// confirmed it stopped. Backup and restore use this to avoid a queued serve +// command racing the filesystem operation. +func (s *RuntimeSession) StopRuntimeForMaintenance() error { + return s.stopRuntime(false) +} + +func (s *RuntimeSession) stopRuntime(resumeQueue bool) error { s.mu.Lock() root := s.runtimeScroll.Root started := s.started @@ -111,7 +122,7 @@ func (s *RuntimeSession) StopRuntime() error { s.runtimeScroll.LastError = "" err := s.store.UpdateScroll(s.runtimeScroll) s.mu.Unlock() - if err == nil && started { + if err == nil && started && resumeQueue { s.mu.Lock() s.started = true s.mu.Unlock() @@ -131,7 +142,7 @@ func (s *RuntimeSession) Backup(ctx context.Context, artifact string, registryCr s.mu.Lock() root := s.runtimeScroll.Root s.mu.Unlock() - return s.runtimeBackend.BackupRuntime(ctx, root, artifact, registryCredentials) + return s.runtimeBackend.BackupRuntime(ctx, root, artifact, registryCredentials, true) } func (s *RuntimeSession) ApplyRestore(materialized *ports.RuntimeMaterialization) error { @@ -178,6 +189,7 @@ func (s *RuntimeSession) ApplyRestore(materialized *ports.RuntimeMaterialization s.runtimeScroll.Root = root s.runtimeScroll.ScrollName = scrollService.GetCurrent().Name s.runtimeScroll.ScrollYAML = string(scrollYAML) + s.runtimeScroll.InstalledReleaseState = domain.InstalledReleaseReady s.runtimeScroll.Routing = routing s.runtimeScroll.Status = domain.RuntimeScrollStatusStopped s.runtimeScroll.LastError = "" diff --git a/apps/druid/core/services/runtime_supervisor.go b/apps/druid/core/services/runtime_supervisor.go index 6a63a23b..2cfa7461 100644 --- a/apps/druid/core/services/runtime_supervisor.go +++ b/apps/druid/core/services/runtime_supervisor.go @@ -27,8 +27,10 @@ type RuntimeSupervisor struct { workerCallbackURL string workerTimeout time.Duration - mu sync.Mutex - sessions map[string]*RuntimeSession + mu sync.Mutex + sessions map[string]*RuntimeSession + operationsMu sync.Mutex + operations map[string]*sync.Mutex } type EnsureOptions struct { @@ -65,6 +67,7 @@ func NewRuntimeSupervisor( manager: manager, workerTimeout: 20 * time.Minute, sessions: map[string]*RuntimeSession{}, + operations: map[string]*sync.Mutex{}, } runtimeBackend, err := backendFactory.Create(supervisor) if err != nil { @@ -144,13 +147,14 @@ func (s *RuntimeSupervisor) createWithOwner(artifact string, name string, ownerI return nil, err } placeholder := &domain.RuntimeScroll{ - ID: id, - OwnerID: ownerID, - Artifact: artifact, - Root: s.runtimeBackend.RootRef(id, namespace), - Status: domain.RuntimeScrollStatusCreated, - Procedures: domain.ProcedureStatusMap{}, - ReservedPorts: fixedDeveloperPorts(), + InstalledReleaseState: domain.InstalledReleasePending, + ID: id, + OwnerID: ownerID, + Artifact: artifact, + Root: s.runtimeBackend.RootRef(id, namespace), + Status: domain.RuntimeScrollStatusCreated, + Procedures: domain.ProcedureStatusMap{}, + ReservedPorts: fixedDeveloperPorts(), } if err := s.store.CreateScroll(placeholder); err != nil { return nil, err @@ -179,6 +183,8 @@ func (s *RuntimeSupervisor) createWithOwner(artifact string, name string, ownerI func (s *RuntimeSupervisor) Ensure(options EnsureOptions) (*domain.RuntimeScroll, error) { id := coreservices.RuntimeScrollIDFromName(options.Name) if id != "" { + unlock := s.lockRuntimeOperation(id) + defer unlock() runtimeScroll, err := s.store.GetScroll(id) if err == nil { if options.Namespace != "" && runtimeScroll.Root != "" { @@ -193,6 +199,10 @@ func (s *RuntimeSupervisor) Ensure(options EnsureOptions) (*domain.RuntimeScroll if artifact == "" { artifact = runtimeScroll.Artifact } + runtimeScroll.InstalledReleaseState = domain.InstalledReleasePending + if err := s.store.UpdateScroll(runtimeScroll); err != nil { + return nil, err + } materialized, err := s.materializeNewScroll(context.Background(), s.runtimeBackend, artifact, id, options.Namespace, options.RegistryCredentials) if err != nil { runtimeScroll.Status = domain.RuntimeScrollStatusError @@ -208,15 +218,9 @@ func (s *RuntimeSupervisor) Ensure(options EnsureOptions) (*domain.RuntimeScroll if runtimeScroll.Status == domain.RuntimeScrollStatusError && (options.Artifact == "" || options.Artifact == runtimeScroll.Artifact) { return s.persistEnsureOptions(runtimeScroll, options) } - if options.Artifact != "" { - nextDigest := resolveArtifactDigest(options.Artifact, options.RegistryCredentials) - artifactChanged := options.Artifact != runtimeScroll.Artifact - digestChanged := nextDigest != "" && nextDigest != runtimeScroll.ArtifactDigest - if artifactChanged || digestChanged { - applyEnsureOptions(runtimeScroll, options) - return s.updateExistingScroll(runtimeScroll, options.Artifact, nextDigest, options.RegistryCredentials, false) - } - } + // Reconciliation does not accept releases on the user's behalf. In + // particular it must neither follow a moved tag nor undo a restore. + // Existing workloads change only through the explicit Update operation. return s.persistEnsureOptions(runtimeScroll, options) } if !errors.Is(err, domain.ErrRuntimeScrollNotFound) { @@ -281,6 +285,7 @@ func (s *RuntimeSupervisor) applyMaterializedScroll(runtimeScroll *domain.Runtim runtimeScroll.Root = materialized.Root runtimeScroll.ScrollName = scroll.Name runtimeScroll.ScrollYAML = string(materialized.ScrollYAML) + runtimeScroll.InstalledReleaseState = domain.InstalledReleaseReady runtimeScroll.Status = domain.RuntimeScrollStatusCreated runtimeScroll.LastError = "" runtimeScroll.Procedures = domain.ProcedureStatusMap{} diff --git a/apps/druid/core/services/runtime_supervisor_test.go b/apps/druid/core/services/runtime_supervisor_test.go index 46d01199..59a881c1 100644 --- a/apps/druid/core/services/runtime_supervisor_test.go +++ b/apps/druid/core/services/runtime_supervisor_test.go @@ -488,6 +488,21 @@ commands: } } +func TestRuntimeSessionStopForMaintenanceKeepsQueueQuiescent(t *testing.T) { + session := newRuntimeSessionForTest(t, map[string]domain.LockStatus{}, updatedScrollYAML("maintenance")) + session.Start() + + if err := session.StopRuntimeForMaintenance(); err != nil { + t.Fatal(err) + } + session.mu.Lock() + started := session.started + session.mu.Unlock() + if started { + t.Fatal("maintenance stop should keep the runtime queue paused") + } +} + func TestRuntimeSupervisorStartDoesNotHydrateStoppedScroll(t *testing.T) { store := newTestStateStore(t) runtimeScroll := &domain.RuntimeScroll{ @@ -919,7 +934,7 @@ func TestRuntimeSupervisorEnsureDoesNotRetryExistingError(t *testing.T) { Artifact: "registry.local/invalid:1.0", Root: store.Root("invalid-scroll"), ScrollName: "invalid-scroll", - ScrollYAML: cachedScrollYAML("start"), + ScrollYAML: updatedScrollYAML("backup-worker"), Status: domain.RuntimeScrollStatusError, LastError: "procedure field mode is unsupported", Procedures: domain.ProcedureStatusMap{}, @@ -945,7 +960,7 @@ func TestRuntimeSupervisorEnsureDoesNotRetryExistingError(t *testing.T) { } } -func TestRuntimeSupervisorEnsureUpdatesChangedArtifact(t *testing.T) { +func TestRuntimeSupervisorEnsureDoesNotApplyUnacceptedArtifact(t *testing.T) { store := newTestStateStore(t) root := "k8s://druid/druid-update-scroll-data" existing := &domain.RuntimeScroll{ @@ -953,7 +968,7 @@ func TestRuntimeSupervisorEnsureUpdatesChangedArtifact(t *testing.T) { Artifact: "registry.local/lab:1.0", Root: root, ScrollName: "old-scroll", - ScrollYAML: cachedScrollYAML("start"), + ScrollYAML: updatedScrollYAML("backup-recovery"), Status: domain.RuntimeScrollStatusRunning, Procedures: domain.ProcedureStatusMap{ "start": {"start.0": {Status: domain.ScrollLockStatusDone}}, @@ -984,26 +999,11 @@ func TestRuntimeSupervisorEnsureUpdatesChangedArtifact(t *testing.T) { t.Fatal(err) } - if backend.stopRoot != root { - t.Fatalf("stop root = %s, want %s", backend.stopRoot, root) + if backend.spawnCount != 0 || backend.stopRoot != "" { + t.Fatalf("reconciliation mutated installed runtime: worker=%#v stop=%s", backend.action, backend.stopRoot) } - if backend.action.Mode != ports.RuntimeWorkerModeUpdate || backend.action.Artifact != "registry.local/lab:2.0" || backend.action.RootRef != root { - t.Fatalf("worker action = %#v", backend.action) - } - if updated.Artifact != "registry.local/lab:2.0" || updated.ScrollName != "updated-scroll" { - t.Fatalf("updated scroll = %#v", updated) - } - if updated.Status != domain.RuntimeScrollStatusStopped { - t.Fatalf("status = %s, want stopped", updated.Status) - } - if len(updated.Procedures) != 0 { - t.Fatalf("procedures = %#v, want cleared", updated.Procedures) - } - if len(updated.Routing) != 1 || updated.Routing[0].PortName != "main" { - t.Fatalf("routing = %#v, want matching route preserved", updated.Routing) - } - if !strings.Contains(updated.ScrollYAML, "updated-scroll") { - t.Fatalf("scroll yaml = %q", updated.ScrollYAML) + if updated.Artifact != existing.Artifact || updated.ScrollYAML != existing.ScrollYAML || updated.Status != domain.RuntimeScrollStatusRunning { + t.Fatalf("reconciliation changed installed release: %#v", updated) } } @@ -1015,7 +1015,7 @@ func TestRuntimeSupervisorUpdateUsesPullWorkerWhenAvailable(t *testing.T) { Artifact: "registry.local/lab:1.0", Root: root, ScrollName: "old-scroll", - ScrollYAML: cachedScrollYAML("start"), + ScrollYAML: updatedScrollYAML("restore-recovery"), Status: domain.RuntimeScrollStatusStopped, Procedures: domain.ProcedureStatusMap{}, } @@ -1031,19 +1031,20 @@ func TestRuntimeSupervisorUpdateUsesPullWorkerWhenAvailable(t *testing.T) { ) supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") - updated, err := supervisor.Ensure(EnsureOptions{Artifact: "registry.local/lab:2.0", Name: "update-worker"}) + accepted := "registry.local/lab@sha256:" + strings.Repeat("a", 64) + updated, err := supervisor.Update("update-worker", accepted, nil, nil) if err != nil { t.Fatal(err) } - if backend.action.Mode != ports.RuntimeWorkerModeUpdate || backend.action.RootRef != root { + if backend.action.Mode != ports.RuntimeWorkerModeUpdate || backend.action.RootRef != root || backend.action.Artifact != accepted { t.Fatalf("worker action = %#v", backend.action) } - if updated.Artifact != "registry.local/lab:2.0" || updated.ArtifactDigest != "sha256:updated" || updated.ScrollName != "updated-worker" { + if updated.Artifact != accepted || updated.ArtifactDigest != "sha256:updated" || updated.ScrollName != "updated-worker" { t.Fatalf("updated scroll = %#v", updated) } } -func TestRuntimeSupervisorUpdateRefreshesCurrentArtifactAndRestartsRunningScroll(t *testing.T) { +func TestRuntimeSupervisorUpdateAppliesAcceptedDigestAndRestartsRunningScroll(t *testing.T) { store := newTestStateStore(t) root := "runtime://refresh-worker" existing := &domain.RuntimeScroll{ @@ -1063,7 +1064,8 @@ func TestRuntimeSupervisorUpdateRefreshesCurrentArtifactAndRestartsRunningScroll supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") - updated, err := supervisor.Update("refresh-worker", "", nil) + accepted := "registry.local/lab@sha256:" + strings.Repeat("b", 64) + updated, err := supervisor.Update("refresh-worker", accepted, nil, nil) if err != nil { t.Fatal(err) } @@ -1074,7 +1076,7 @@ func TestRuntimeSupervisorUpdateRefreshesCurrentArtifactAndRestartsRunningScroll if backend.stopRoot != root { t.Fatalf("stop root = %s, want %s", backend.stopRoot, root) } - if backend.action.Mode != ports.RuntimeWorkerModeUpdate || backend.action.Artifact != "registry.local/lab:1.0" { + if backend.action.Mode != ports.RuntimeWorkerModeUpdate || backend.action.Artifact != accepted { t.Fatalf("worker action = %#v", backend.action) } if updated.Status != domain.RuntimeScrollStatusRunning { @@ -1137,6 +1139,239 @@ func TestRuntimeSupervisorRestoreUsesPullWorkerResult(t *testing.T) { } } +func TestRuntimeSupervisorBackupStopsAndRestartsRunningScroll(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "backup-worker", + Artifact: "registry.local/lab:1.0", + Root: "runtime://backup-worker", + ScrollName: "backup-worker", + ScrollYAML: strings.Replace(cachedScrollYAML("start"), "run: once", "run: persistent", 1), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{procedureStatusUpdates: []ports.ProcedureStatusUpdate{{Procedure: "start.0", Status: domain.ScrollLockStatusRunning}}} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + t.Cleanup(func() { + if _, err := supervisor.Stop(runtimeScroll.ID); err != nil { + t.Errorf("stop test runtime: %v", err) + } + }) + + backedUp, err := supervisor.Backup("backup-worker", "registry.local/backups:1", nil) + if err != nil { + t.Fatal(err) + } + if backend.stopRoot != runtimeScroll.Root { + t.Fatalf("stop root = %q, want %q", backend.stopRoot, runtimeScroll.Root) + } + if backend.backupRoot != runtimeScroll.Root || backend.backupArtifact != "registry.local/backups:1" || !backend.backupPreserveReleaseManifest { + t.Fatalf("backup = root=%q artifact=%q preserve=%t", backend.backupRoot, backend.backupArtifact, backend.backupPreserveReleaseManifest) + } + if backedUp.Status != domain.RuntimeScrollStatusRunning { + t.Fatalf("backup status = %s, want running", backedUp.Status) + } +} + +func TestRuntimeSupervisorBackupFailureKeepsPriorRunningScrollStopped(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "backup-recovery", + Artifact: "registry.local/lab:1.0", + Root: "runtime://backup-recovery", + ScrollName: "backup-recovery", + ScrollYAML: strings.Replace(cachedScrollYAML("start"), "run: once", "run: persistent", 1), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{backupErr: errors.New("registry unavailable"), procedureStatusUpdates: []ports.ProcedureStatusUpdate{{Procedure: "start.0", Status: domain.ScrollLockStatusRunning}}} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + + if _, err := supervisor.Backup("backup-recovery", "registry.local/backups:1", nil); err == nil { + t.Fatal("backup error = nil, want registry error") + } + recovered, err := store.GetScroll("backup-recovery") + if err != nil { + t.Fatal(err) + } + if recovered.Status != domain.RuntimeScrollStatusError { + t.Fatalf("failed backup status = %s, want error", recovered.Status) + } +} + +func TestRuntimeSupervisorRestoreFailureKeepsPriorRunningScrollStopped(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "restore-recovery", + Artifact: "registry.local/lab:1.0", + Root: "runtime://restore-recovery", + ScrollName: "restore-recovery", + ScrollYAML: strings.Replace(cachedScrollYAML("start"), "run: once", "run: persistent", 1), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{callbacks: callbacks, workerErr: errors.New("backup pull failed"), procedureStatusUpdates: []ports.ProcedureStatusUpdate{{Procedure: "start.0", Status: domain.ScrollLockStatusRunning}}} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") + + if _, err := supervisor.Restore("restore-recovery", "registry.local/backups:1", true, nil); err == nil { + t.Fatal("restore error = nil, want pull error") + } + recovered, err := store.GetScroll("restore-recovery") + if err != nil { + t.Fatal(err) + } + if recovered.Status != domain.RuntimeScrollStatusError { + t.Fatalf("failed restore status = %s, want error", recovered.Status) + } +} + +func TestRuntimeSupervisorUnsafeRestoreFailureKeepsRuntimeStopped(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "restore-unsafe", + Artifact: "registry.local/lab:1.0", + Root: "runtime://restore-unsafe", + ScrollName: "restore-unsafe", + ScrollYAML: updatedScrollYAML("restore-unsafe"), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{callbacks: callbacks, workerErr: errors.New("restore root may be partial: injected rollback failure")} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") + + if _, err := supervisor.Restore("restore-unsafe", "registry.local/backups:1", true, nil); err == nil { + t.Fatal("restore error = nil, want unsafe restore failure") + } + updated, err := store.GetScroll("restore-unsafe") + if err != nil { + t.Fatal(err) + } + if updated.Status == domain.RuntimeScrollStatusRunning { + t.Fatalf("unsafe restore failure must not restart runtime: %#v", updated) + } + if !strings.Contains(updated.LastError, "restore root may be partial") { + t.Fatalf("last error = %q, want unsafe restore detail", updated.LastError) + } +} + +func TestRuntimeSupervisorSerializesBackupWithStartAndEnsure(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "serialized-operation", + Artifact: "registry.local/lab:1.0", + Root: "runtime://serialized-operation", + ScrollName: "serialized-operation", + ScrollYAML: updatedScrollYAML("serialized-operation"), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + backupStarted := make(chan struct{}) + backupRelease := make(chan struct{}) + backend := &fakeWorkerBackend{backupStarted: backupStarted, backupRelease: backupRelease} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + + backupDone := make(chan error, 1) + go func() { + _, err := supervisor.Backup(runtimeScroll.ID, "registry.local/backups:1", nil) + backupDone <- err + }() + select { + case <-backupStarted: + case <-time.After(time.Second): + t.Fatal("backup did not reach its stopped operation") + } + + startDone := make(chan error, 1) + go func() { + _, err := supervisor.StartScroll(runtimeScroll.ID) + startDone <- err + }() + ensureDone := make(chan error, 1) + go func() { + _, err := supervisor.Ensure(EnsureOptions{Name: runtimeScroll.ID}) + ensureDone <- err + }() + for name, done := range map[string]<-chan error{"start": startDone, "ensure": ensureDone} { + select { + case err := <-done: + t.Fatalf("%s completed during backup: %v", name, err) + case <-time.After(100 * time.Millisecond): + } + } + + close(backupRelease) + for name, done := range map[string]<-chan error{"backup": backupDone, "start": startDone, "ensure": ensureDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("%s error = %v", name, err) + } + case <-time.After(time.Second): + t.Fatalf("%s did not complete after backup", name) + } + } +} + +func TestRuntimeSupervisorStartResumesMaintenanceStoppedQueue(t *testing.T) { + store := newTestStateStore(t) + runtimeScroll := &domain.RuntimeScroll{ + ID: "resume-maintenance", + Artifact: "registry.local/lab:1.0", + Root: "runtime://resume-maintenance", + ScrollName: "resume-maintenance", + ScrollYAML: updatedScrollYAML("resume-maintenance"), + Status: domain.RuntimeScrollStatusRunning, + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + var runs atomic.Int32 + backend := &fakeWorkerBackend{runCommand: func(command ports.RuntimeCommand) (*int, error) { + runs.Add(1) + return nil, errors.New("run should not persist in test") + }} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + session, err := supervisor.sessionFor(runtimeScroll.ID) + if err != nil { + t.Fatal(err) + } + if err := session.StopRuntimeForMaintenance(); err != nil { + t.Fatal(err) + } + if _, err := supervisor.StartScroll(runtimeScroll.ID); err != nil { + t.Fatal(err) + } + deadline := time.After(time.Second) + for runs.Load() == 0 { + select { + case <-deadline: + t.Fatal("start did not resume the maintenance-stopped runtime queue") + default: + time.Sleep(10 * time.Millisecond) + } + } +} + func TestNewRuntimeSessionRequiresPersistedScrollYAML(t *testing.T) { store := newTestStateStore(t) runtimeScroll := &domain.RuntimeScroll{ @@ -1374,22 +1609,29 @@ func newRuntimeSessionForTest(t *testing.T, commands map[string]domain.LockStatu } type fakeWorkerBackend struct { - callbacks *WorkerCallbackManager - procedureStatusObserver ports.ProcedureStatusObserver - procedureStatusUpdates []ports.ProcedureStatusUpdate - scrollYAML string - digest string - workerErr error - workerDone <-chan error - action ports.RuntimeWorkerAction - stopRoot string - deleteRoot string - spawnCount int - runCommand func(ports.RuntimeCommand) (*int, error) - stopRuntime func(string) error - console io.ReadWriteCloser - consoleRoot string - consoleProcedure string + callbacks *WorkerCallbackManager + procedureStatusObserver ports.ProcedureStatusObserver + procedureStatusUpdates []ports.ProcedureStatusUpdate + scrollYAML string + digest string + workerErr error + workerDone <-chan error + workerResult *ports.RuntimeWorkerResult + action ports.RuntimeWorkerAction + stopRoot string + deleteRoot string + backupRoot string + backupArtifact string + backupPreserveReleaseManifest bool + backupErr error + backupStarted chan struct{} + backupRelease <-chan struct{} + spawnCount int + runCommand func(ports.RuntimeCommand) (*int, error) + stopRuntime func(string) error + console io.ReadWriteCloser + consoleRoot string + consoleProcedure string } func newRuntimeSupervisorForTest(t *testing.T, store ports.RuntimeScrollStore, manager *coreservices.RuntimeScrollManager, factory ports.RuntimeBackendFactory) *RuntimeSupervisor { @@ -1474,8 +1716,17 @@ func (f *fakeWorkerBackend) DeleteRuntime(root string, purgeData bool) error { return nil } -func (f *fakeWorkerBackend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error { - return nil +func (f *fakeWorkerBackend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error { + f.backupRoot = root + f.backupArtifact = artifact + f.backupPreserveReleaseManifest = preserveReleaseManifest + if f.backupStarted != nil { + close(f.backupStarted) + } + if f.backupRelease != nil { + <-f.backupRelease + } + return f.backupErr } func (f *fakeWorkerBackend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { @@ -1493,10 +1744,14 @@ func (f *fakeWorkerBackend) SpawnPullWorker(ctx context.Context, action ports.Ru if f.callbacks == nil { return done, nil } - err := f.callbacks.Complete(action.RuntimeID, ports.RuntimeWorkerResult{ + result := ports.RuntimeWorkerResult{ ScrollYAML: f.scrollYAML, ArtifactDigest: f.digest, - }) + } + if f.workerResult != nil { + result = *f.workerResult + } + err := f.callbacks.Complete(action.RuntimeID, result) return done, err } diff --git a/apps/druid/core/services/runtime_update.go b/apps/druid/core/services/runtime_update.go index 22360826..4430eb9f 100644 --- a/apps/druid/core/services/runtime_update.go +++ b/apps/druid/core/services/runtime_update.go @@ -3,6 +3,7 @@ package services import ( "context" "errors" + "regexp" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" @@ -11,19 +12,36 @@ import ( "go.uber.org/zap" ) -func (s *RuntimeSupervisor) Update(id string, artifact string, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) { +var ErrUnacceptedUpdate = errors.New("update requires an explicitly accepted sha256 artifact reference") +var ErrInstalledReleaseChanged = errors.New("installed release changed; check updates again") +var acceptedUpdateReference = regexp.MustCompile(`^[^@\s]+@sha256:[a-f0-9]{64}$`) + +func (s *RuntimeSupervisor) Update(id string, artifact string, registryCredentials []domain.RegistryCredential, expectedDigest *string) (*domain.RuntimeScroll, error) { + if !acceptedUpdateReference.MatchString(artifact) { + return nil, ErrUnacceptedUpdate + } + unlock := s.lockRuntimeOperation(id) + defer unlock() runtimeScroll, err := s.store.GetScroll(id) if err != nil { return nil, err } - if artifact == "" { - artifact = runtimeScroll.Artifact + if expectedDigest != nil { + if !acceptedUpdateReference.MatchString("expected@" + *expectedDigest) { + return nil, ErrUnacceptedUpdate + } + installed, err := s.installedRelease(context.Background(), runtimeScroll) + if err != nil { + return nil, err + } + if installed["digest"] != *expectedDigest { + return nil, ErrInstalledReleaseChanged + } } - knownDigest := resolveArtifactDigest(artifact, registryCredentials) - return s.updateExistingScroll(runtimeScroll, artifact, knownDigest, registryCredentials, true) + return s.updateExistingScroll(runtimeScroll, artifact, registryCredentials) } -func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeScroll, artifact string, knownDigest string, registryCredentials []domain.RegistryCredential, restartIfRunning bool) (*domain.RuntimeScroll, error) { +func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeScroll, artifact string, registryCredentials []domain.RegistryCredential) (*domain.RuntimeScroll, error) { wasRunning := runtimeScroll.Status == domain.RuntimeScrollStatusRunning existingRouting := make([]domain.RuntimeRouteAssignment, len(runtimeScroll.Routing)) copy(existingRouting, runtimeScroll.Routing) @@ -52,6 +70,10 @@ func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeSc _ = s.store.UpdateScroll(runtimeScroll) return nil, err } + runtimeScroll.InstalledReleaseState = domain.InstalledReleasePending + if err := s.store.UpdateScroll(runtimeScroll); err != nil { + return nil, err + } materialized, err := s.runPullWorker(context.Background(), s.runtimeBackend, ports.RuntimeWorkerModeUpdate, runtimeScroll.ID, artifact, runtimeScroll.Root, registryCredentials, "") if err != nil { runtimeScroll.Status = domain.RuntimeScrollStatusError @@ -72,9 +94,6 @@ func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeSc runtimeScroll.Artifact = artifact } runtimeScroll.ArtifactDigest = materialized.ArtifactDigest - if runtimeScroll.ArtifactDigest == "" { - runtimeScroll.ArtifactDigest = knownDigest - } runtimeScroll.Root = materialized.Root runtimeScroll.ScrollName = scroll.Name runtimeScroll.ScrollYAML = string(materialized.ScrollYAML) @@ -87,6 +106,7 @@ func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeSc return nil, err } runtimeScroll.Routing = preserveRoutingAssignments(existingRouting, ports) + runtimeScroll.InstalledReleaseState = domain.InstalledReleaseReady runtimeScroll.LastError = "" if wasRunning || runtimeScroll.Status == domain.RuntimeScrollStatusStopped { runtimeScroll.Status = domain.RuntimeScrollStatusStopped @@ -99,8 +119,8 @@ func (s *RuntimeSupervisor) updateExistingScroll(runtimeScroll *domain.RuntimeSc if err := s.publishDeclaredPrivateUI(runtimeScroll, scroll); err != nil { return nil, err } - if wasRunning && restartIfRunning { - return s.StartScroll(runtimeScroll.ID) + if wasRunning { + return s.startScroll(runtimeScroll.ID) } return s.store.GetScroll(runtimeScroll.ID) } diff --git a/apps/druid/core/services/runtime_update_test.go b/apps/druid/core/services/runtime_update_test.go new file mode 100644 index 00000000..7bcd8d3f --- /dev/null +++ b/apps/druid/core/services/runtime_update_test.go @@ -0,0 +1,53 @@ +package services + +import ( + "errors" + "strings" + "testing" + + "github.com/highcard-dev/daemon/internal/core/domain" + coreservices "github.com/highcard-dev/daemon/internal/core/services" +) + +func TestRuntimeUpdateRejectsUnacceptedReferenceBeforeStopping(t *testing.T) { + for _, artifact := range []string{"", "registry.local/example:latest", "registry.local/example@sha256:bad"} { + t.Run(artifact, func(t *testing.T) { + store := newTestStateStore(t) + original := &domain.RuntimeScroll{ID: "accepted-update", Root: "runtime://accepted-update", Artifact: "registry.local/example:v1", ScrollYAML: cachedScrollYAML("start"), Status: domain.RuntimeScrollStatusRunning} + if err := store.CreateScroll(original); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + _, err := supervisor.Update(original.ID, artifact, nil, nil) + if err == nil || !strings.Contains(err.Error(), "accepted sha256") { + t.Fatalf("error = %v, want accepted digest validation", err) + } + if backend.stopRoot != "" || backend.spawnCount != 0 { + t.Fatal("invalid selection stopped or modified the workload") + } + }) + } +} + +func TestUpdateRejectsChangedInstalledDescriptorBeforeStopping(t *testing.T) { + store := newTestStateStore(t) + fixture := &domain.RuntimeScroll{ID: "stale-update", Root: "runtime://fixture", Artifact: "registry.local/deployment:v1", ScrollYAML: cachedScrollYAML("start"), Status: domain.RuntimeScrollStatusRunning} + fixture.InstalledReleaseState = domain.InstalledReleaseReady + fixture.ArtifactDigest = "sha256:" + strings.Repeat("b", 64) + if err := store.CreateScroll(fixture); err != nil { + t.Fatal(err) + } + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: "name: registry.local/reusable\n", digest: "sha256:" + strings.Repeat("b", 64)} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") + expected := "sha256:" + strings.Repeat("a", 64) + _, err := supervisor.Update(fixture.ID, "registry.local/deployment@sha256:"+strings.Repeat("c", 64), nil, &expected) + if !errors.Is(err, ErrInstalledReleaseChanged) { + t.Fatalf("stale update error: %v", err) + } + if backend.stopRoot != "" || backend.spawnCount != 0 { + t.Fatal("stale acceptance mutated runtime") + } +} diff --git a/docs/runtime-lifecycle-verification.md b/docs/runtime-lifecycle-verification.md new file mode 100644 index 00000000..4251b310 --- /dev/null +++ b/docs/runtime-lifecycle-verification.md @@ -0,0 +1,72 @@ +# Runtime lifecycle acceptance + +## Current scope (2026-09-29, local only) + +This feature is evaluated against master `bc17ea075910c3390402e3323fec243975f5ee2b`. +Only Unified Scroll Lifecycle changes and problems introduced by those changes +belong in this diff. Pre-existing general bugs are deferred, not claimed fixed. + +The approved scope pruning is complete locally. Broad operation/attempt IDs, +durable admission/recovery stores, worker-completion/cleanup protocol changes, +snapshot ownership metadata, generic local materialization-copy changes, and +UI-package persistence repairs have been removed. Ordinary Stop/requeue and +platform-wide Docker host-gateway changes have also been excluded from the PR +delta. Late semantic validation, failed-stop Update retry, and queue-drain timeout +repairs remain deferred because those problems already exist on master. + +## Retained contract + +- Explicit updates require an accepted immutable digest, with an optional + expected-installed-digest precondition checked under the maintenance lock. +- Reconciliation does not silently update an installed release. The read-only + installed-release endpoint reads the volume's actual descriptor. +- Updates stage the candidate, remove obsolete unprotected files, and preserve + paths protected by either installed or candidate declarations. Globs expand + against both trees. Protected-copy modes, numeric owners and symlinks survive + the new staging path; Docker/Kubernetes update helpers do not recursively + overwrite those owners afterward. +- Removed protection declarations protect this transition, not indefinitely + across future releases. No hidden persisted protection list was introduced. +- Backups include runtime-created files and preserve the installed descriptor. + Successful maintenance can restart the workload. Failed backup/restore leaves + it stopped with an error; error text is not proof that restarting is safe. +- Command admission shares the maintenance lock, but waiting does not hold it. +- Explicit source timestamps and stable layer ordering support reproducible + Team publication. The semantic-validator binary supports Core lifecycle APIs. + +## Current verification + +After pruning: + +- `go test ./... -timeout=180s`: passes. +- Rebuilt-image `TestDockerUpdatePreservesProtectedOwner`: passes (15.77s), + including custom-owner workload access, symlinks, permission preservation, + new-data default ownership and failure before replacement when chown fails. +- Focused protected-copy and failure-restart tests: three race-enabled runs pass. +- Rebuilt-image `TestKubernetesBackendCLIComplexLifecycle`: passes (102.77s) + after pruning. Exercises a real workload in a disposable namespace, accepted + digest despite tag movement, stale-baseline rejection, protected data, complete + backup restoration and byte-identical installed descriptor. It does not run + the browser/Core/engine/operator chain. +- Earlier broader crash/recovery evidence does not apply to the narrowed code. + +## Remaining acceptance limits + +This is not a claim of full product merge readiness. Team workflow cutover, +full browser-to-workload acceptance and CI on the final local changes are still +outstanding. No shared services, production deployments, existing workloads, +remote branches or PRs were changed by this pruning pass. + +## Workspace recovery (2026-10-03) + +The temporary worktree and its temporary backup archives were lost. The retained +September 29 source/test state was reconstructed from successful session file +changes and complete file reads, on top of commit `a9f5dd9`. It now lives in +`druid-local/.worktrees/scroll-lifecycle/druid-cli` on the local branch +`recovery/scroll-lifecycle-20261003`. Reconstruction tooling and the recovered +schema are under `druid-local/.recovery/lifecycle-20261003`. + +The recovered full Go suite passes. Docker/Kubernetes results above are +historical September 29 evidence, not fresh acceptance of this recovered tree. +The temporary pre-pruning archives are unavailable; do not rely on their old +paths as backups. Deferred behavior remains excluded as described above. diff --git a/internal/api/generated.go b/internal/api/generated.go index 914574b4..7d08a312 100644 --- a/internal/api/generated.go +++ b/internal/api/generated.go @@ -229,9 +229,12 @@ type RuntimeUIPackages map[string]RuntimeUIPackage // UpdateScrollRequest defines model for UpdateScrollRequest. type UpdateScrollRequest struct { - // Artifact Optional target artifact. If omitted, the daemon refreshes the runtime's current artifact. - Artifact *string `json:"artifact,omitempty"` - RegistryCredentials *[]RegistryCredential `json:"registry_credentials,omitempty"` + // Artifact Explicitly accepted immutable target. Missing or mutable tag references are rejected before stopping the runtime. + Artifact string `json:"artifact"` + + // ExpectedInstalledDigest Optional precondition checked against the actual installed descriptor under the maintenance lock. + ExpectedInstalledDigest *string `json:"expected_installed_digest,omitempty"` + RegistryCredentials *[]RegistryCredential `json:"registry_credentials,omitempty"` } // RunScrollCommandParams defines parameters for RunScrollCommand. @@ -379,6 +382,9 @@ type ClientInterface interface { // GetScrollQueue request GetScrollQueue(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetInstalledRelease request + GetInstalledRelease(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // RestoreScrollWithBody request with any body RestoreScrollWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -592,6 +598,18 @@ func (c *Client) GetScrollQueue(ctx context.Context, id string, reqEditors ...Re return c.Client.Do(req) } +func (c *Client) GetInstalledRelease(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetInstalledReleaseRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) RestoreScrollWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewRestoreScrollRequestWithBody(c.Server, id, contentType, body) if err != nil { @@ -1184,6 +1202,40 @@ func NewGetScrollQueueRequest(server string, id string) (*http.Request, error) { return req, nil } +// NewGetInstalledReleaseRequest generates requests for GetInstalledRelease +func NewGetInstalledReleaseRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/scrolls/%s/release", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewRestoreScrollRequest calls the generic RestoreScroll builder with application/json body func NewRestoreScrollRequest(server string, id string, body RestoreScrollJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -1600,6 +1652,9 @@ type ClientWithResponsesInterface interface { // GetScrollQueueWithResponse request GetScrollQueueWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollQueueResponse, error) + // GetInstalledReleaseWithResponse request + GetInstalledReleaseWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstalledReleaseResponse, error) + // RestoreScrollWithBodyWithResponse request with any body RestoreScrollWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*RestoreScrollResponse, error) @@ -1898,6 +1953,31 @@ func (r GetScrollQueueResponse) StatusCode() int { return 0 } +type GetInstalledReleaseResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *struct { + Digest string `json:"digest"` + Repository string `json:"repository"` + } +} + +// Status returns HTTPResponse.Status +func (r GetInstalledReleaseResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetInstalledReleaseResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type RestoreScrollResponse struct { Body []byte HTTPResponse *http.Response @@ -2206,6 +2286,15 @@ func (c *ClientWithResponses) GetScrollQueueWithResponse(ctx context.Context, id return ParseGetScrollQueueResponse(rsp) } +// GetInstalledReleaseWithResponse request returning *GetInstalledReleaseResponse +func (c *ClientWithResponses) GetInstalledReleaseWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetInstalledReleaseResponse, error) { + rsp, err := c.GetInstalledRelease(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetInstalledReleaseResponse(rsp) +} + // RestoreScrollWithBodyWithResponse request with arbitrary body returning *RestoreScrollResponse func (c *ClientWithResponses) RestoreScrollWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*RestoreScrollResponse, error) { rsp, err := c.RestoreScrollWithBody(ctx, id, contentType, body, reqEditors...) @@ -2629,6 +2718,35 @@ func ParseGetScrollQueueResponse(rsp *http.Response) (*GetScrollQueueResponse, e return response, nil } +// ParseGetInstalledReleaseResponse parses an HTTP response from a GetInstalledReleaseWithResponse call +func ParseGetInstalledReleaseResponse(rsp *http.Response) (*GetInstalledReleaseResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetInstalledReleaseResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest struct { + Digest string `json:"digest"` + Repository string `json:"repository"` + } + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + } + + return response, nil +} + // ParseRestoreScrollResponse parses an HTTP response from a RestoreScrollWithResponse call func ParseRestoreScrollResponse(rsp *http.Response) (*RestoreScrollResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2875,6 +2993,9 @@ type ServerInterface interface { // Get runtime queue state // (GET /api/v1/scrolls/{id}/queue) GetScrollQueue(c *fiber.Ctx, id string) error + // Read the installed release descriptor from the runtime volume + // (GET /api/v1/scrolls/{id}/release) + GetInstalledRelease(c *fiber.Ctx, id string) error // Execute runtime restore // (POST /api/v1/scrolls/{id}/restore) RestoreScroll(c *fiber.Ctx, id string) error @@ -3084,6 +3205,22 @@ func (siw *ServerInterfaceWrapper) GetScrollQueue(c *fiber.Ctx) error { return siw.Handler.GetScrollQueue(c, id) } +// GetInstalledRelease operation middleware +func (siw *ServerInterfaceWrapper) GetInstalledRelease(c *fiber.Ctx) error { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", c.Params("id"), &id, runtime.BindStyledParameterOptions{Explode: false, Required: true}) + if err != nil { + return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter id: %w", err).Error()) + } + + return siw.Handler.GetInstalledRelease(c, id) +} + // RestoreScroll operation middleware func (siw *ServerInterfaceWrapper) RestoreScroll(c *fiber.Ctx) error { @@ -3265,6 +3402,8 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/api/v1/scrolls/:id/queue", wrapper.GetScrollQueue) + router.Get(options.BaseURL+"/api/v1/scrolls/:id/release", wrapper.GetInstalledRelease) + router.Post(options.BaseURL+"/api/v1/scrolls/:id/restore", wrapper.RestoreScroll) router.Post(options.BaseURL+"/api/v1/scrolls/:id/routing", wrapper.ApplyScrollRouting) @@ -3286,54 +3425,59 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xbX3MbNw7/KhzezdzLWnKuTR98T65z7blNJz47mTy0GQ1FQhKrXZIhuZZ1Hn33G/7Z", - "1f7hSpZiN3anL4klkiDwAwiAIHSPqSyUFCCswWf32NAFFMT/ea5Uvr6WpeVifg2fSzDWfa20VKAtBz+J", - "GMPnoqiWcwuF/+PvGmb4DP9tvCU/jrTH16WwvABHGs7r9XiTYbtWgM8w0Zqs8WaTYQ2fS66B4bNfW1t9", - "qufK6e9A/eILDcTCDdUyz4f51ZbPCPUjDAzVXFkuBT7D7y4uUTWKNMxAg6CApEa5pCRHxhNGitgFzjDc", - "kULlgdmwxoyYLjkbzedjC8b6f87cP7jm1VjNxdzxylmfgTegNFBigSGSc2LQTGokSAEj9M7PcUxYMs0B", - "6YAg4mwcJlzOkCy4tcAyZBeAGIFCCjQHAZpYMIgIxNmoxfjvcmpSvDmKCXgeh4V/hTFuVE7WXjpkLM9z", - "RGUBBs20LCLSozUp8odzbBShCbZ/LqegBbj961ke2Ip/DUaWmoIZocu5kBoYmq6RkOKksXRK6BIEM6PU", - "7nIlQE9SGo2GjvwMxBkqDTC/Oy2NlQXokxmhXMyRdmcBkdIupOb/I259ci8Nc26sXk+oBgbCcpIfcO7i", - "4ot67f4zVx2X1IF7AzlYYOHE9Y9aQKQngrHEln7CVrEsUOpL3GGHuymRQIqjfwtT6kNcQI+7anDC+Dyu", - "Hji8g+fmL/N8Hub5HyC5XVyDUVIY6NtBIVlCIxel1iAsWvjVKBgb8nObrkguU/IrLecajOmTvYojSIGm", - "ICyZBz3nkjCHsGPM4+oc3Ezqglh8hme5JC5+FOSOF2WBz16dnma44CJ8Oq1ZEGUxBR2Pl7YTRmxCto8L", - "EE3f7Of6Y1fv6BaeOKvAGRZlnjtfj8+sLmHf2fQQpfTwVtLlTX3o2zqAO24nNCpiYD8uLMyDcDkxdhJU", - "MqELIuZ+Xc08F/a7b3FqYcPpCIfcr1iXQjgxMsyk8LrVWmqc4RXhLuNpiDIgcKSZ5CqFw5XUCW/U0tAh", - "XkVFcrVtfPf69TevG9bxKgWE0tJKKvMmFAtrFc78fz68UvepZGo/BJ65yEqDdlJ6LSkw5509UL8Q5X0x", - "YzzkFVdtHz3w/S7/0bCzTYKBPkflNOdm8eHyitAlmcNgwPAp346EyIebEy2lPdGQE8tvAY1WxBQ+WRyh", - "NzAjZW4NshIpzW+JhTHjxo6JUmGe1Eg5bmj7+1EyIPYESTjOngwLORDMFDFmJXU6pJUG9IABdizB028s", - "aBBOWUMMPefRf7+rvN9xQXso7LAAPD6bkdxA9nhhyG3pnWeDnamUORBxWIyKODjXMOQip7IULLVP5kGf", - "cJXExI9VPqLvB5YA6jznt/Bek9mM0yQN79gItfyW2/WE2JazbUaKw71W0jMFB5Fe1/Bbff3fTaZrG+B6", - "SDDwGVWSku2h0YA7Dh60V7VGLnfTXHHB5CrN0yHSDTjoGtu+s86ihW2FrxHaYbHdy3sislvnCvJd9nl4", - "wJsMj+4ykOBcdxyHUudpH7dLfi7m74meQ0L6h90FDjkd+4Q/9uwYyIFaqXdF3b5JdlExoG85hcnDgsWA", - "VU4G0okO+R1WOXQV3Rk9qK8bsYP828AV0DvMkEimhptXsWEd7o1QiVQqRCTQt8C8lT/81uWzUr+csHci", - "X3eS723Ak3Ig+IaT8OjVvwyHxGrY6vtJfVQlzhrpvbFSKf9dleFX1YZPCcWWfKJCOvhQQer80aedpWIH", - "GlOqxFHba8S9jUW2vXo0bLe1944zUvM7nOj2UTlYqh0utSmtm5ThWFM9kP+jLwo9IFIe7YNn5vhqcnU7", - "sD5C1JXlwTqphpkGswDjv4zln38YRGM9oibwteouHYS806el5nZ94wjFZBWIBn1eBjsKn36ozOWnj+/9", - "6Wvi9NPH98jKJYhQ+uWeA7tGSstbzkB703fkXdrkyW3l9/dWx5lbX+3ZJn+zkNqeuDyXoc8l6HW1mdTo", - "I0xvJF2CRVQKAbSqvnC30E/GVTYSttjuTBT/GRwsLhSImXQbUylsMIVNV8g3uuQMXby9RDkpBV34YjhD", - "BRHOjJFfyQXoE1/IY9VTA1Eq5zRUhTKU8yX8Jua+Yu4cvTYZYsSSKTFgMk9wBdNqbPSbZ5dbX62qGcAZ", - "dqOBrdPRq9Gpj0sKBFEcn+Fv/FfhRHqFjoni49tX41AOc9/EfKct4Y9gK0NuFc6wJx7udpcsTAx1Oa8v", - "H7V8ec5v9s/T0wrJmFM2IBj/bkKJJNjtPqvuVP+8qto8hxk+0rw+/eYP3PgmZDOoFOSW8FDy8uepLAqi", - "1xHOLo6WzI2/aQdNZDjgjT+5pZWaguWYhp7a8L/lxt7EOV8I/iHBPuZlfbfSw6aqSVeCtHFx7NelcVPL", - "UUETR5rYuGzSJIBoPhbiEJPA2O8lWz+aIaTeIzftAOhSrU1PD68ejYUO/PvgRlX+1EY9CNLBfTfsfZMc", - "g3+b8TE0qZHm280TaST1PPQgjZx+NY0E1LoaCYJ0NILgjhsbQouM6Ue+DkV+c7C67jnbBD/vkuW+usLj", - "X60uRTQpwIJ2W9yHGBrTuhhCfWLbBjprgNbNEz89oRLaD5f7lVBdGDYZ/vb02+GHtDhdSItmvqbS1lrY", - "9qBzlKXd+I9gXybyB5r/lyLu4ugXui13DsYuLyvVsO/63o8/sUoe3x/uq8I/N98YYEZuaTyQba94B7Rs", - "HLCotaM0TmVREMHM+D7+tRnW/nUpAs8XYepTWECWJELrDQ+i1HkVJtz6C5G/eAbVA0ORNrKyBhxNYSa1", - "b0pQUjAu5qOB+5JZC4qbXHSfYnqPJl/V64TLPuv6ii/0Ptel6IborcKOskkx4/PB3L4OChdh3jMMDd0a", - "Qk8RV0Sb7QU4Ctz36So17VhMjczBPAjVMPMZ4pqugrUKucOYV4KhJaxDe9G2MN+HPr52GyqVdxI1KEeA", - "X1eqdyN/JUMC++xgP+QK3HjiPfgajBxQVSXg0VOiFvWj9Pi5hBL26/G/ftoLy1hTjy19fXnRPIawA+/P", - "jVlboD9HWCrAh3HWYKzcdYe+DhP+SkSf+pYScH5wJlop7qjT1XhgS2vdN+rHqkac+3JUn/qVwXNT91B6", - "2NL5FWjDjY2dnFKfhN8rAIutXUjXuukbgX/y3msC4/CY9ICQ2WpPePGxs91s8ZDwGRagCq9EGhN+xRA7", - "pyvd1AuO0FHdD5Y+pDdu+E9atLkJfcS7z4ef9CjVGGOl2gW0VH9anH0rwz6cpepmeCupl7kkzKDVgueA", - "VGgWcRbPiCXHqaHk42arxG6H1Hi1f5laabZ5JO6toZcYGPpwiSpU3FXK35NSN9jEAvTh+q35Yl2M7/2e", - "m3HcYvikRKY7CvrjqlcBm110qr6e2DWNq06+VGf6E+UnQ13im5ikPJN3ohW3CxQbaJomVYAl/oh3kpUg", - "FSICkVwDYeuTaclzi0KnwIdL9PH85peKypE2qapfoaTNr9lh84IS1lRj0Nc2hqepXwaq3Vgy43nsX+le", - "ZFO2EXtDK60O9cWcX13i2DKGx9ipLhLtNfQEJkLrTOEbo8S2Vg3+3uVmNpxMhKL3Q6+4JlzKtwS3S8PF", - "PFEwTzUOoVArX4LvIYoUVjA1fmaCypXUFnERmum4FDWkZYOACt2Z98nWFUQXQJcmuTB2ifSX/lLmlp9E", - "XVaqTUlfabNP4k1o9Mn5DOia5unl0QT6q39wCciKWLpw6YfjncEt5FJ5bcYf2lX4uWkJGudCSBtQc+aI", - "CKVgGtKTetzgzafN/wMAAP//k6kvPuU+AAA=", + "H4sIAAAAAAAC/+xbW3PbtrP/Khicvh1ZctokM/V5qev04jaZ+NjJ5CFxNRCwklCBAAKAtvX36Lv/BxdS", + "FAnKluK0dqcvbSwAi8VvF3vD8hZTVWglQTqLj26xpXMoSPjnsdZiea5Kx+XsHD6XYJ3/WRulwTgOYRKx", + "ls9kUS3nDorwj28MTPER/p/Rmvwo0R6dl9LxAjxpOK7X49UAu6UGfISJMWSJV6sBNvC55AYYPvq4sdVl", + "PVdN/gQaFp8YIA4uqFFC9PNrHJ8SGkYYWGq4dlxJfITfnpyiahQZmIIBSQEpg4SiRCAbCCNN3BwPMNyQ", + "QovIbFxjh8yUnA1ns5ED68J/jvx/cM2rdYbLmeeVsy4Dr0AboMQBQ0RwYtFUGSRJAUP0NszxTDgyEYBM", + "RBBxNooTTqdIFdw5YAPk5oAYgUJJNAMJhjiwiEjE2XCD8T/VxOZ48xQz8DwMC/8Xx7jVgizD6ZB1XAhE", + "VQEWTY0qEtLDJSnE/Tm2mtAM27+XEzAS/P71rABsxb8Bq0pDwQ7R6UwqAwxNlkgqedBYOiF0AZLZYW53", + "dS3BjHMSTYqOwgzEGSotsLA7La1TBZiDKaFczpDxdwGR0s2V4f8hfn12LwMzbp1ZjqkBBtJxIna4d2nx", + "Sb327jtXXZfchXsFAhyweOO6Vy0i0jmCdcSVYcJasCxS6p64xQ73UxKBHEc/SVuaXUxAh7tqcMz4LK3u", + "uby99+Zf9Xwc6vkrEOHm52C1kha6elAolpHISWkMSIfmYTWKyobC3KYpUovc+bVRMwPWdsmepRGkwVCQ", + "jsyinIUizCPsGQu4egM3VaYgDh/hqVDE+4+C3PCiLPDRs8PDAS64jH8d1izIspiASdfLuDEjLnO2D3OQ", + "Tdsc5oZrV+/oFx54rcADLEshvK3HR86UcNfdDBDl5PBa0cVFfek3ZQA33I1pEkTPflw6mMXDCWLdOIpk", + "TOdEzsK6mnku3cvnOLewYXSkR+4jNqWU/hgDzJQMsjVGGTzA14T7iKdxlJ4DJ5pZrnI4nCmTsUYbEtrF", + "quhErtaNly9efPeioR3PckBoo5yiSjShmDun8SD8L7hX6v8qmb4bgsBcYqVBO3t6oygwb50DUG+IDraY", + "MR7jirNNG93z+zb70dCzVYaBLkflRHA7f396RuiCzKDXYYSQb0tAFNzNgVHKHRgQxPErQMNrYosQLA7R", + "K5iSUjiLnELa8CviYMS4dSOidZynDNKeG7r5+zDrEDsHyRjOzhnmqseZaWLttTJ5l1ZaMD0K2NKEQL+x", + "oEE4pw3J9Rwn+/22sn77Oe0+t8Mi8PhoSoSFwcO5Ib9lMJ4NdiZKCSByNx+VcPCmoc9ETlQpWW6fQQB9", + "zHUWkzBW2YiuHVgA6GPBr+CdIdMpp1kawbAR6vgVd8sxcRvGtukpdrdaWcsUDUR+XcNudeV/M54sXYTr", + "Ps4gRFRZSq6DRgPuNLjTXtUatdhO85pLpq7zPO1yuh4DXWPbNdaDpGHrw9cIbdHYdvKe8ezOmwKxTT93", + "d3jj/tFtChKN65brUBqRt3Hbzs/l7B0xM8ic/n65wC63467D73t3LAigTpltXrerkm1ULJgrTmF8P2fR", + "o5XjnnCiRX6LVvalolu9Bw11I7aTfetJAYPBjIFkbriZivXL8E4PlQmlokcCcwUsaPn9s64QlYblhL2V", + "YtkKvtcOT6ke5xtvwoNX/wY4Blb9Wt8N6pMo8aAR3luntA6/VRF+VW24zAi25GMdw8H7HqSOH0PYWWq2", + "ozLlShy1vibcN7EYrFOPhu5u7L3ljtT89ge6XVR2PtUWk9o8rZ80wKmmuiP/eycKHSByFu19YGbvavJP", + "N1pwyp1YIkIpaAcM8aIoY/HUBbcxRG+4tSH7N2g9NFtXny0iBpABzxQwNIGpMoCCRvtlPpVPhaJhhNH7", + "XHyE//j4xw+fPtnL//3Bzsm3L14efSQH08OD7y9vXz5ffZOTFtzosMeYS+uIEMAaFbCexEcboEpG/BGd", + "A10AQ2RGPInAG6GuJALVJFFFSBlUSgYmzCqI93OSSApIKLpoHeW+J3hEtafgEGlpuFte+A1SIA/EgDku", + "4x2Lf/1cXaXfPrwLlqkJ9G8f3iGnFiBjWZwHztwSaaOuOAMTzIIn70PKQG6NS8jp/RH8+mrPTfIXc2Xc", + "gc8BGPpcgllWmymDPsDkQtEFOESVlECryhT3C8NkXEVqcYv1zkTz38HD5d2knCq/MVXSxWuyah/ylSk5", + "QyevT5EgpaTz8FDAUEGkv+IorOQSzEEocrLqGYZof79ixWyABF/AJzkLrwneCRo7QIw4MiEW7CAQvIZJ", + "NTb8FNjlLlTyagbwAPvRyNbh8NnwMPhsDZJojo/wd+GnaK2CQEdE89HVs1EsFfpfUiy4ecJfwFUFt42i", + "Ig7EY957yuLEWLMM8goePZQuw2bfHh5WSKZ4uwHB6E8by0dRn+/S9lZlNIhqk+c4I3jhF4ff/YUbX8RI", + "D5WSXBEey4HhPpVFQcwywdnG0ZGZDVWIKIkBjnjjS7+0ElPUHNuQ0yb8r7l1F2nOF4K/SyCUYtauuelg", + "U9Xrq4Ns4uLZr58NbH2OCpo00sTGR9o2A0TzIRVHowfW/ajY8sEUIfdWu9q0sD4MXXXk8OzBWGjBfxfc", + "qIotN1GPB2nhvh32rkqOILxbhfgiK5Hmu9ZXkkju6exeEjn82yQSUWtLJB6kJREEN9y66FpUil/EMj6A", + "2J3FdcvZKtp5n0h0xRUfRmtxaWJIAQ6M3+I2+tAU8iYXGoL+TaAHDdDaMfTlVxTC5qPu3UKokqnVAD8/", + "fN7/yJimS+XQNNSbNqUWt93pHg3yZvwXcE8T+R3V/0sR9370C82WvwcjH5eVut92/RjGv7JIHt4e3vVC", + "8dhsY4QZ+aXpQm5axRugZeOCJantJXGqioJIZke36V+rfumflzLyfBKnfg0NGGSJ0HrDnSi1XswJdyEh", + "Cql2FD0wlGgjp2rAq+Q8agDjcjbsyZfsUlLc5KL9TNV5UPpbrU4shLC2rfhC63NeyraLXgtsL52UUz7r", + "je1rp3AS5z1C19CuIXQEcUaMXSfA6cBdm65z0/bF1CoB9l6oxpmPENd8hXCjyN2PeXUwtIBlbL1aP1p0", + "oU+dAJYqHYxEDcoe4NdV/O3In6kYwD462HdJgRvP3zunwcgDVVUCHjwk2qC+lxw/l1DC3XL8/zDtiUWs", + "uYeorrzC0QKGsAXvz41Za6A/J1gqwPtxNiCA2K1In1Z16PM09xHC3WoO629CNaCV5U6Z5d2vLI25g4rm", + "5T2sXw0YokQqySkRaE0rJNOCWIdsSSlYOy39cIAWpV2+NEoAwkLotX5CqDdYPyWE6njjLQRdKVEWTTWq", + "ymHb1Mc6ta0Ecx4n/JvHfO0kN+J870SmEtxexrnxdp2XevgGJhXF0tynI/rcBzyPTdx92cWGzM/AWG5d", + "apJW5iB+CgQsdU0iU8umqwShm+ROFRjFJ9l7RFwbnT9PPvTa7GO6T/QVF6AKr0wUHB+y00cJlWzqBXvI", + "qG61zF/SCz/8D635XcQW/e33I0x6kGKedUpvA1rpfyzOoUvoLpyVbicI18oshCLMous5F4B07MPyGs+I", + "I/uJoeSjZhfSdoPUaIh5mlJpdlBlyh6xTR8Yen+KKlR8Jh7S7FwBJLMAvT9/bb9YFqPbsOdqlLbovymJ", + "6ZaA/rriZ8RmG52qZS59kICrJtncRx9fKT7p+wBjlYKUR/LMeM3dHKXetKZKFeBIuOKtYCWeChGJiDBA", + "2PJgUnLhUGw0eX+KPhxfvKmo7KmTuvrAK69+zea1JxSw5nrunkrA6hPdw26ieywRZFoBL349/vbFy9z3", + "5tyi+rS7Z89+wffdBaed9Dl+p8aQ5X5TnzxHnULXxFa9fC3Fjidv+78pF6llq127yelzahWvNLGvFez4", + "7BSnDlI8wl7dEtFOD1tkInaLFSBd4KR6noGQK/qZDcOYxNX57jOtiXWoNcH10liLyrwR5XrlUHweWkBo", + "m0sUrmFiw8wMlTNlHOIy9tZyJWtIywYBHZu1b7PdWlFsNrswNUZ1l74pheMHSZaVaHOnr6TZJfEq9rYJ", + "PgW6pCK/PKlAd/XPPmi6Jo7OfcjkeWdwBULpIM303W2Fn5+WoXEspXIRNa+O4ZrZxulJPW7x6nL13wAA", + "AP//pU/ObfRCAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/core/domain/runtime_scroll.go b/internal/core/domain/runtime_scroll.go index 3e03f221..effc21f4 100644 --- a/internal/core/domain/runtime_scroll.go +++ b/internal/core/domain/runtime_scroll.go @@ -20,22 +20,33 @@ const ( RuntimeScrollStatusDeleted RuntimeScrollStatus = "deleted" ) +// Empty state denotes a legacy record whose digest may describe the source +// artifact rather than the installed release. Pending is persisted before a +// worker can mutate the volume; only its successful result confirms the cache. +type InstalledReleaseState string + +const ( + InstalledReleasePending InstalledReleaseState = "pending" + InstalledReleaseReady InstalledReleaseState = "ready" +) + type RuntimeScroll struct { - ID string `json:"id"` - OwnerID string `json:"owner_id,omitempty"` - Artifact string `json:"artifact"` - ArtifactDigest string `json:"artifact_digest,omitempty"` - Root string `json:"root"` - ScrollName string `json:"scroll_name"` - ScrollYAML string `json:"-"` - Status RuntimeScrollStatus `json:"status"` - LastError string `json:"last_error,omitempty"` - Routing []RuntimeRouteAssignment `json:"routing,omitempty"` - UIPackages RuntimeUIPackages `json:"ui_packages,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - Procedures ProcedureStatusMap `json:"procedures,omitempty"` - ReservedPorts []Port `json:"reserved_ports,omitempty"` + InstalledReleaseState InstalledReleaseState `json:"-"` + ID string `json:"id"` + OwnerID string `json:"owner_id,omitempty"` + Artifact string `json:"artifact"` + ArtifactDigest string `json:"artifact_digest,omitempty"` + Root string `json:"root"` + ScrollName string `json:"scroll_name"` + ScrollYAML string `json:"-"` + Status RuntimeScrollStatus `json:"status"` + LastError string `json:"last_error,omitempty"` + Routing []RuntimeRouteAssignment `json:"routing,omitempty"` + UIPackages RuntimeUIPackages `json:"ui_packages,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Procedures ProcedureStatusMap `json:"procedures,omitempty"` + ReservedPorts []Port `json:"reserved_ports,omitempty"` } type RuntimeState struct { diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index 9967deea..b3020f5c 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -41,7 +41,7 @@ type RuntimeBackendInterface interface { StopCommand(root string, command string) error StopRuntime(root string) error DeleteRuntime(root string, purgeData bool) error - BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error + BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error SpawnPullWorker(ctx context.Context, action RuntimeWorkerAction) (<-chan error, error) OpenConsole(ctx context.Context, root string, procedure string) (io.ReadWriteCloser, error) Signal(commandName string, target string, signal string, root string) error @@ -159,6 +159,7 @@ const ( RuntimeWorkerModeCreate RuntimeWorkerMode = "create" RuntimeWorkerModeUpdate RuntimeWorkerMode = "update" RuntimeWorkerModeRestore RuntimeWorkerMode = "restore" + RuntimeWorkerModeInspect RuntimeWorkerMode = "inspect" ) type RuntimeWorkerAction struct { @@ -170,6 +171,7 @@ type RuntimeWorkerAction struct { MountPath string CallbackURL string TokenFile string + NewDataOwner string RegistryCredentials []domain.RegistryCredential } diff --git a/internal/core/services/registry/oci.go b/internal/core/services/registry/oci.go index 9d9dbdc6..40831189 100644 --- a/internal/core/services/registry/oci.go +++ b/internal/core/services/registry/oci.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "regexp" + "sort" "strconv" "strings" "sync/atomic" @@ -162,6 +163,10 @@ func (c *OciClient) Pull(dir string, artifact string) error { } func (c *OciClient) PullSelective(dir string, artifact string, includeData bool, progress *domain.SnapshotProgress) error { + return c.PullSelectiveWithOptions(dir, artifact, includeData, progress, TransferOptions{}) +} + +func (c *OciClient) PullSelectiveWithOptions(dir string, artifact string, includeData bool, progress *domain.SnapshotProgress, options TransferOptions) error { repo, ref, _ := utils.ParseArtifactRef(artifact) if repo == "" || ref == "" { @@ -334,8 +339,19 @@ func (c *OciClient) PullSelective(dir string, artifact string, includeData bool, annotations := fullDesc.Annotations fileName := filepath.Join(dir, "manifest.json") - err = os.WriteFile(fileName, jsonData, 0644) - if err != nil { + if options.PreserveReleaseManifest { + preserved, readErr := os.ReadFile(fileName) + if readErr != nil { + return fmt.Errorf("preserve-release-manifest requires manifest.json in artifact: %w", readErr) + } + var descriptor v1.Descriptor + if err := json.Unmarshal(preserved, &descriptor); err != nil || descriptor.Digest.String() == "" { + if err == nil { + err = fmt.Errorf("descriptor digest is empty") + } + return fmt.Errorf("preserve-release-manifest requires a valid manifest.json in artifact: %w", err) + } + } else if err = os.WriteFile(fileName, jsonData, 0644); err != nil { return fmt.Errorf("failed to write manifest descriptor: %w", err) } @@ -746,6 +762,10 @@ func copyToRegistry(ctx context.Context, fs *file.Store, repo *remote.Repository } func (c *OciClient) Push(folder string, repo string, tag string, overrides map[string]string, packMeta bool, scrollFile *domain.File) (v1.Descriptor, error) { + return c.PushWithOptions(folder, repo, tag, overrides, packMeta, scrollFile, TransferOptions{}) +} + +func (c *OciClient) PushWithOptions(folder string, repo string, tag string, overrides map[string]string, packMeta bool, scrollFile *domain.File, options TransferOptions) (v1.Descriptor, error) { ctx := context.Background() // Authenticate before doing any expensive local work. @@ -777,6 +797,23 @@ func (c *OciClient) Push(folder string, repo string, tag string, overrides map[s if len(fsFileNames) == 0 { return v1.Descriptor{}, fmt.Errorf("no files found to push") } + if options.PreserveReleaseManifest { + manifest, err := os.ReadFile(filepath.Join(folder, "manifest.json")) + if err != nil { + return v1.Descriptor{}, fmt.Errorf("preserve-release-manifest requires manifest.json: %w", err) + } + var descriptor v1.Descriptor + if err := json.Unmarshal(manifest, &descriptor); err != nil || descriptor.Digest.String() == "" { + if err == nil { + err = fmt.Errorf("descriptor digest is empty") + } + return v1.Descriptor{}, fmt.Errorf("preserve-release-manifest requires a valid manifest.json: %w", err) + } + fsFileNames = append(fsFileNames, "manifest.json") + } + // Map iteration order must not change the manifest digest of an identical + // release. These root paths are disjoint, so sorting preserves extraction. + sort.Strings(fsFileNames) fs, err := c.newFileStore(folder) if err != nil { @@ -804,7 +841,9 @@ func (c *OciClient) Push(folder string, repo string, tag string, overrides map[s dataExists, _ := utils.FileExists(dataDir) if dataExists { var explicitChunks []*domain.Chunks - if scrollFile != nil { + // A backup is a complete runtime snapshot, not the release's curated + // chunk selection. Include files created since installation as well. + if scrollFile != nil && !options.PreserveReleaseManifest { explicitChunks = scrollFile.Chunks } chunks, err := utils.AutoChunkDataDir(dataDir, explicitChunks) diff --git a/internal/core/services/registry/oci_test.go b/internal/core/services/registry/oci_test.go index 2179b3d0..07b59313 100644 --- a/internal/core/services/registry/oci_test.go +++ b/internal/core/services/registry/oci_test.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -11,8 +12,53 @@ import ( "github.com/highcard-dev/daemon/internal/core/domain" ocidigest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" ) +func TestPushIdenticalReleaseHasStableDigest(t *testing.T) { + folder := t.TempDir() + if err := os.WriteFile(filepath.Join(folder, "scroll.yaml"), []byte("name: fixture\napp_version: '1.0.0'\n"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(folder, ".meta"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(folder, ".meta", "en-US.md"), []byte("---\nname: Fixture\n---\nFixture.\n"), 0644); err != nil { + t.Fatal(err) + } + srv := fakeRegistry(t) + client := &OciClient{credentialStore: NewCredentialStore(nil), plainHTTP: true} + repo := strings.TrimPrefix(srv.URL, "http://") + "/test/reproducible" + var first v1.Descriptor + var firstLayers []string + for i := 0; i < 32; i++ { + tag := fmt.Sprintf("build-%d", i) + desc, err := client.Push(folder, repo, tag, map[string]string{"org.opencontainers.image.created": "1970-01-01T00:00:00Z"}, true, nil) + if err != nil { + t.Fatal(err) + } + response, err := http.Get(srv.URL + "/v2/test/reproducible/manifests/" + tag) + if err != nil { + t.Fatal(err) + } + var manifest v1.Manifest + err = json.NewDecoder(response.Body).Decode(&manifest) + response.Body.Close() + if err != nil { + t.Fatal(err) + } + var layers []string + for _, layer := range manifest.Layers { + layers = append(layers, layer.Annotations["org.opencontainers.image.title"]+"@"+layer.Digest.String()) + } + if i == 0 { + first, firstLayers = desc, layers + } else if desc.Digest != first.Digest { + t.Fatalf("identical rebuild changed digest: first layers %v; rebuild layers %v", firstLayers, layers) + } + } +} + // fakeRegistry returns a plain-HTTP httptest server that implements the bare // minimum of the OCI Distribution spec so that oras.Copy can complete a push. func fakeRegistry(t *testing.T) *httptest.Server { @@ -294,6 +340,70 @@ func TestPushPullExecutableDataChunkPreservesMode(t *testing.T) { } } +func TestPreserveReleaseManifestRoundTrip(t *testing.T) { + t.Setenv("DRUID_REGISTRY_PLAIN_HTTP", "true") + srv := fakeRegistry(t) + registryHost := strings.TrimPrefix(srv.URL, "http://") + folder := t.TempDir() + if err := os.WriteFile(filepath.Join(folder, "scroll.yaml"), []byte("name: test\nversion: 0.1.0\napp_version: test\n"), 0644); err != nil { + t.Fatal(err) + } + want := []byte(`{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","size":123}`) + if err := os.WriteFile(filepath.Join(folder, "manifest.json"), want, 0644); err != nil { + t.Fatal(err) + } + client := &OciClient{credentialStore: NewCredentialStore(nil), plainHTTP: true} + repo := registryHost + "/test/preserve-release" + if err := os.MkdirAll(filepath.Join(folder, "data"), 0755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"declared.txt", "runtime-created.txt"} { + if err := os.WriteFile(filepath.Join(folder, "data", name), []byte(name), 0644); err != nil { + t.Fatal(err) + } + } + scrollFile := &domain.File{Chunks: []*domain.Chunks{{Name: "declared", Path: "declared.txt"}}} + if _, err := client.PushWithOptions(folder, repo, "backup", nil, false, scrollFile, TransferOptions{PreserveReleaseManifest: true}); err != nil { + t.Fatalf("push backup: %v", err) + } + pullDir := t.TempDir() + if err := client.PullSelectiveWithOptions(pullDir, repo+":backup", true, nil, TransferOptions{PreserveReleaseManifest: true}); err != nil { + t.Fatalf("restore backup: %v", err) + } + got, err := os.ReadFile(filepath.Join(pullDir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if string(got) != string(want) { + t.Fatalf("restored release manifest = %s, want %s", got, want) + } + for _, name := range []string{"declared.txt", "runtime-created.txt"} { + got, err := os.ReadFile(filepath.Join(pullDir, "data", name)) + if err != nil || string(got) != name { + t.Fatalf("backup omitted runtime data %s: %s (%v)", name, got, err) + } + } +} + +func TestPreserveReleaseManifestRequiresSnapshotDescriptor(t *testing.T) { + t.Setenv("DRUID_REGISTRY_PLAIN_HTTP", "true") + srv := fakeRegistry(t) + registryHost := strings.TrimPrefix(srv.URL, "http://") + folder := t.TempDir() + if err := os.WriteFile(filepath.Join(folder, "scroll.yaml"), []byte("name: test\nversion: 0.1.0\napp_version: test\n"), 0644); err != nil { + t.Fatal(err) + } + client := &OciClient{credentialStore: NewCredentialStore(nil), plainHTTP: true} + repo := registryHost + "/test/missing-release" + if _, err := client.Push(folder, repo, "release", nil, false, nil); err != nil { + t.Fatalf("push normal release: %v", err) + } + err := client.PullSelectiveWithOptions(t.TempDir(), repo+":release", true, nil, TransferOptions{PreserveReleaseManifest: true}) + if err == nil || !strings.Contains(err.Error(), "preserve-release-manifest requires manifest.json") { + t.Fatalf("preserving missing descriptor error = %v", err) + } +} + func TestFetchFileReadsScrollYAMLDescriptor(t *testing.T) { tmpDir := t.TempDir() t.Chdir(tmpDir) diff --git a/internal/core/services/registry/transfer_options.go b/internal/core/services/registry/transfer_options.go new file mode 100644 index 00000000..3bb7731e --- /dev/null +++ b/internal/core/services/registry/transfer_options.go @@ -0,0 +1,8 @@ +package registry + +// TransferOptions controls whether a runtime release descriptor travels with a +// snapshot. Normal Scroll publication excludes manifest.json because it +// describes a local installation, not the artifact being published. +type TransferOptions struct { + PreserveReleaseManifest bool +} diff --git a/internal/runtime/backend_factory_test.go b/internal/runtime/backend_factory_test.go index 8608a1bd..15d0bb59 100644 --- a/internal/runtime/backend_factory_test.go +++ b/internal/runtime/backend_factory_test.go @@ -162,7 +162,7 @@ func (f fakeBackend) DeleteRuntime(root string, purgeData bool) error { return nil } -func (f fakeBackend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error { +func (f fakeBackend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error { return nil } diff --git a/internal/runtime/docker/state_store.go b/internal/runtime/docker/state_store.go index cbbbb9f8..846c00ef 100644 --- a/internal/runtime/docker/state_store.go +++ b/internal/runtime/docker/state_store.go @@ -25,6 +25,7 @@ const scrollsTableSQL = ` owner_id TEXT NOT NULL DEFAULT '', artifact TEXT NOT NULL, artifact_digest TEXT NOT NULL DEFAULT '', + installed_release_state TEXT NOT NULL DEFAULT '', root TEXT NOT NULL, scroll_name TEXT NOT NULL, scroll_yaml TEXT NOT NULL DEFAULT '', @@ -95,9 +96,9 @@ func (s *StateStore) CreateScroll(scroll *domain.RuntimeScroll) error { return err } _, err = db.Exec(` - INSERT INTO scrolls (id, owner_id, artifact, artifact_digest, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `, scroll.ID, scroll.OwnerID, scroll.Artifact, scroll.ArtifactDigest, scroll.Root, scroll.ScrollName, scroll.ScrollYAML, scroll.Status, scroll.LastError, formatTime(scroll.CreatedAt), formatTime(scroll.UpdatedAt), string(procedures), string(routing), string(reservedPorts), string(uiPackages)) + INSERT INTO scrolls (id, owner_id, artifact, artifact_digest, installed_release_state, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, scroll.ID, scroll.OwnerID, scroll.Artifact, scroll.ArtifactDigest, scroll.InstalledReleaseState, scroll.Root, scroll.ScrollName, scroll.ScrollYAML, scroll.Status, scroll.LastError, formatTime(scroll.CreatedAt), formatTime(scroll.UpdatedAt), string(procedures), string(routing), string(reservedPorts), string(uiPackages)) if err != nil { return fmt.Errorf("create runtime scroll %s: %w", scroll.ID, err) } @@ -112,7 +113,7 @@ func (s *StateStore) ListScrolls() ([]*domain.RuntimeScroll, error) { defer db.Close() rows, err := db.Query(` - SELECT id, owner_id, artifact, artifact_digest, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json + SELECT id, owner_id, artifact, artifact_digest, installed_release_state, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json FROM scrolls ORDER BY id `) @@ -140,7 +141,7 @@ func (s *StateStore) GetScroll(id string) (*domain.RuntimeScroll, error) { defer db.Close() row := db.QueryRow(` - SELECT id, owner_id, artifact, artifact_digest, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json + SELECT id, owner_id, artifact, artifact_digest, installed_release_state, root, scroll_name, scroll_yaml, status, last_error, created_at, updated_at, procedures_json, routing_json, reserved_ports_json, ui_packages_json FROM scrolls WHERE id = ? `, id) @@ -177,9 +178,9 @@ func (s *StateStore) UpdateScroll(scroll *domain.RuntimeScroll) error { } res, err := db.Exec(` UPDATE scrolls - SET owner_id = ?, artifact = ?, artifact_digest = ?, root = ?, scroll_name = ?, scroll_yaml = ?, status = ?, last_error = ?, updated_at = ?, procedures_json = ?, routing_json = ?, reserved_ports_json = ?, ui_packages_json = ? + SET owner_id = ?, artifact = ?, artifact_digest = ?, installed_release_state = ?, root = ?, scroll_name = ?, scroll_yaml = ?, status = ?, last_error = ?, updated_at = ?, procedures_json = ?, routing_json = ?, reserved_ports_json = ?, ui_packages_json = ? WHERE id = ? - `, scroll.OwnerID, scroll.Artifact, scroll.ArtifactDigest, scroll.Root, scroll.ScrollName, scroll.ScrollYAML, scroll.Status, scroll.LastError, formatTime(scroll.UpdatedAt), string(procedures), string(routing), string(reservedPorts), string(uiPackages), scroll.ID) + `, scroll.OwnerID, scroll.Artifact, scroll.ArtifactDigest, scroll.InstalledReleaseState, scroll.Root, scroll.ScrollName, scroll.ScrollYAML, scroll.Status, scroll.LastError, formatTime(scroll.UpdatedAt), string(procedures), string(routing), string(reservedPorts), string(uiPackages), scroll.ID) if err != nil { return err } @@ -250,6 +251,10 @@ func (s *StateStore) open() (*sql.DB, error) { db.Close() return nil, err } + if err := ensureColumn(db, "scrolls", "installed_release_state", "TEXT NOT NULL DEFAULT ''"); err != nil { + db.Close() + return nil, err + } if err := ensureColumn(db, "scrolls", "root", "TEXT NOT NULL DEFAULT ''"); err != nil { db.Close() return nil, err @@ -346,7 +351,7 @@ func scanRuntimeScroll(scanner runtimeScrollScanner) (*domain.RuntimeScroll, err var routingJSON string var reservedPortsJSON string var uiPackagesJSON string - if err := scanner.Scan(&scroll.ID, &scroll.OwnerID, &scroll.Artifact, &scroll.ArtifactDigest, &scroll.Root, &scroll.ScrollName, &scroll.ScrollYAML, &status, &lastError, &createdAt, &updatedAt, &proceduresJSON, &routingJSON, &reservedPortsJSON, &uiPackagesJSON); err != nil { + if err := scanner.Scan(&scroll.ID, &scroll.OwnerID, &scroll.Artifact, &scroll.ArtifactDigest, &scroll.InstalledReleaseState, &scroll.Root, &scroll.ScrollName, &scroll.ScrollYAML, &status, &lastError, &createdAt, &updatedAt, &proceduresJSON, &routingJSON, &reservedPortsJSON, &uiPackagesJSON); err != nil { return nil, err } scroll.Status = domain.RuntimeScrollStatus(status) diff --git a/internal/runtime/docker/state_store_test.go b/internal/runtime/docker/state_store_test.go index 87acedbd..e3493197 100644 --- a/internal/runtime/docker/state_store_test.go +++ b/internal/runtime/docker/state_store_test.go @@ -1,12 +1,56 @@ package docker import ( + "database/sql" "path/filepath" + "strings" "testing" "github.com/highcard-dev/daemon/internal/core/domain" ) +func TestStateStoreMigratesAndPersistsInstalledReleaseState(t *testing.T) { + dir := t.TempDir() + db, err := sql.Open("sqlite", filepath.Join(dir, "state.db")) + if err != nil { + t.Fatal(err) + } + // Model the previous schema without the confirmation marker. + _, err = db.Exec(strings.Replace(scrollsTableSQL, "installed_release_state TEXT NOT NULL DEFAULT '',", "", 1)) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(`INSERT INTO scrolls (id, artifact, artifact_digest, root, scroll_name, status, created_at, updated_at) VALUES ('legacy', 'backup:v1', 'old-digest', 'root', 'scroll', 'stopped', '', '')`) + if err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + store, err := NewStateStore(dir) + if err != nil { + t.Fatal(err) + } + legacy, err := store.GetScroll("legacy") + if err != nil || legacy.InstalledReleaseState != "" || legacy.ArtifactDigest != "old-digest" { + t.Fatalf("migration = %#v, %v", legacy, err) + } + for _, state := range []domain.InstalledReleaseState{domain.InstalledReleasePending, domain.InstalledReleaseReady} { + legacy.InstalledReleaseState = state + if err := store.UpdateScroll(legacy); err != nil { + t.Fatal(err) + } + got, err := store.GetScroll(legacy.ID) + if err != nil || got.InstalledReleaseState != state { + t.Fatalf("state = %#v, %v", got, err) + } + all, err := store.ListScrolls() + if err != nil || len(all) != 1 || all[0].InstalledReleaseState != state { + t.Fatalf("listed state = %#v, %v", all, err) + } + } +} + func TestStateStorePersistsProcedureStatuses(t *testing.T) { store, err := NewStateStore(t.TempDir()) if err != nil { diff --git a/internal/runtime/docker/workers.go b/internal/runtime/docker/workers.go index dbacedf8..a3b2069b 100644 --- a/internal/runtime/docker/workers.go +++ b/internal/runtime/docker/workers.go @@ -16,15 +16,19 @@ import ( "github.com/highcard-dev/daemon/internal/core/ports" ) -func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error { +func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error { if artifact == "" { return fmt.Errorf("backup artifact is required") } - return b.runWorkerRootCommand(ctx, root, []string{ + command := []string{ "worker", "push", "--artifact", artifact, "--root", "/scroll", - }, registryCredentials) + } + if preserveReleaseManifest { + command = append(command, "--preserve-release-manifest") + } + return b.runWorkerRootCommand(ctx, root, command, registryCredentials) } func (b *Backend) runWorkerRootCommand(ctx context.Context, root string, command []string, registryCredentials []domain.RegistryCredential) error { @@ -109,8 +113,22 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke if err := b.pullImage(ctx, b.config.WorkerImage); err != nil { return nil, err } - if err := b.prepareWritableRoot(ctx, root); err != nil { - return nil, err + workerUser := "" + if action.Mode == ports.RuntimeWorkerModeUpdate { + // Staging protected data must retain its original ownership and modes. + // Do not chmod the installed tree before it has been copied. + image, _, err := b.client.ImageInspectWithRaw(ctx, b.config.WorkerImage) + if err != nil { + return nil, err + } + if image.Config != nil { + action.NewDataOwner = image.Config.User + } + workerUser = "0" + } else if action.Mode != ports.RuntimeWorkerModeInspect { + if err := b.prepareWritableRoot(ctx, root); err != nil { + return nil, err + } } registryConfig, err := json.Marshal(struct { Registries []domain.RegistryCredential `json:"registries"` @@ -118,7 +136,7 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke if err != nil { return nil, err } - rootMount, err := DockerMount(root, action.MountPath, false, "") + rootMount, err := DockerMount(root, action.MountPath, action.Mode == ports.RuntimeWorkerModeInspect, "") if err != nil { return nil, err } @@ -146,14 +164,8 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke created, err := b.client.ContainerCreate(ctx, &container.Config{ Image: b.config.WorkerImage, Entrypoint: []string{"druid"}, - Cmd: []string{ - "worker", "pull", - "--artifact", artifact, - "--runtime-id", action.RuntimeID, - "--mode", string(action.Mode), - "--root", action.MountPath, - "--callback-url", action.CallbackURL, - }, + Cmd: workerPullCommand(action, artifact), + User: workerUser, Env: dockerWorkerEnv([]string{ "DRUID_WORKER_TOKEN_FILE=" + action.TokenFile, "DRUID_RUNTIME_REGISTRY_CONFIG_JSON=" + string(registryConfig), @@ -195,3 +207,18 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke }() return done, nil } + +func workerPullCommand(action ports.RuntimeWorkerAction, artifact string) []string { + command := []string{ + "worker", "pull", + "--artifact", artifact, + "--runtime-id", action.RuntimeID, + "--mode", string(action.Mode), + "--root", action.MountPath, + "--callback-url", action.CallbackURL, + } + if action.Mode == ports.RuntimeWorkerModeUpdate && action.NewDataOwner != "" { + command = append(command, "--new-data-owner", action.NewDataOwner) + } + return command +} diff --git a/internal/runtime/kubernetes/resources.go b/internal/runtime/kubernetes/resources.go index 5949af75..314254c6 100644 --- a/internal/runtime/kubernetes/resources.go +++ b/internal/runtime/kubernetes/resources.go @@ -55,7 +55,9 @@ if [ -n "${DRUID_RUNTIME_REGISTRY_CONFIG_JSON:-}" ]; then else druid "$@" fi -chown -R 1000:1000 "$DRUID_WORKER_ROOT"` +if [ "$DRUID_WORKER_MODE" != "update" ]; then + chown -R 1000:1000 "$DRUID_WORKER_ROOT" +fi` ) const ( @@ -85,11 +87,23 @@ func workerPullJobSpec(namespace string, jobName string, pvc string, image strin "--root", action.MountPath, "--callback-url", action.CallbackURL, } + if action.Mode == ports.RuntimeWorkerModeUpdate { + command = append(command, "--new-data-owner", "1000:1000") + } + if action.Mode == ports.RuntimeWorkerModeInspect { + // No credential shell or recursive ownership changes for read-only inspection. + command = append([]string{"druid"}, command[4:]...) + } job := helperJobSpec(namespace, jobName, pvc, image, command, imagePullSecret, map[string]string{ labelComponent: "worker-pull", labelRuntimeID: runtimeLabel(action.RuntimeID), }) container := &job.Spec.Template.Spec.Containers[0] + if action.Mode == ports.RuntimeWorkerModeInspect { + for i := range container.VolumeMounts { + container.VolumeMounts[i].ReadOnly = true + } + } runAsRoot := int64(0) runAsNonRoot := false container.SecurityContext = &corev1.SecurityContext{ @@ -100,6 +114,7 @@ func workerPullJobSpec(namespace string, jobName string, pvc string, image strin container.Env = append(container.Env, corev1.EnvVar{Name: "DRUID_WORKER_TOKEN_FILE", Value: action.TokenFile}, corev1.EnvVar{Name: workerPullRootEnvName, Value: action.MountPath}, + corev1.EnvVar{Name: "DRUID_WORKER_MODE", Value: string(action.Mode)}, ) if registryConfigSecret != "" { container.Env = append(container.Env, corev1.EnvVar{ @@ -128,11 +143,14 @@ func runtimeLabel(runtimeID string) string { return dnsLabel(runtimeID) } -func backupJobSpec(namespace string, jobName string, pvc string, image string, artifact string, imagePullSecret string, registryConfigSecret string, registryPlainHTTP bool) *batchv1.Job { +func backupJobSpec(namespace string, jobName string, pvc string, image string, artifact string, imagePullSecret string, registryConfigSecret string, registryPlainHTTP bool, preserveReleaseManifest bool) *batchv1.Job { command := []string{"druid", "push", artifact, "/scroll"} if registryConfigSecret != "" { command = []string{"sh", "-c", registryConfigScript, "sh", "push", artifact, "/scroll"} } + if preserveReleaseManifest { + command = append(command, "--preserve-release-manifest") + } job := helperJobSpec(namespace, jobName, pvc, image, command, imagePullSecret, map[string]string{ labelComponent: "backup", }) diff --git a/internal/runtime/kubernetes/resources_test.go b/internal/runtime/kubernetes/resources_test.go index ae58d818..750fac4b 100644 --- a/internal/runtime/kubernetes/resources_test.go +++ b/internal/runtime/kubernetes/resources_test.go @@ -454,7 +454,7 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { assertFinishedJobTTL(t, job) container := job.Spec.Template.Spec.Containers[0] command := strings.Join(container.Command, " ") - for _, want := range []string{"druid --config /tmp/druid-registry.json", "worker pull", "--mode update", "--runtime-id deployment-123", "--callback-url", "chown -R 1000:1000"} { + for _, want := range []string{"druid --config /tmp/druid-registry.json", "worker pull", "--mode update", "--runtime-id deployment-123", "--callback-url", "--new-data-owner 1000:1000", `if [ "$DRUID_WORKER_MODE" != "update" ]; then`} { if !strings.Contains(command, want) { t.Fatalf("command = %#v, want %s", container.Command, want) } @@ -472,6 +472,9 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { if env[workerPullRootEnvName] != "/scroll" { t.Fatalf("%s = %q, want /scroll", workerPullRootEnvName, env[workerPullRootEnvName]) } + if env["DRUID_WORKER_MODE"] != "update" { + t.Fatalf("update must skip recursive ownership changes: %#v", container.Env) + } if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil || *container.SecurityContext.RunAsUser != 0 { t.Fatalf("worker pull must run as root to repair PVC ownership, securityContext = %#v", container.SecurityContext) } @@ -486,6 +489,21 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { } } +func TestInstalledReleaseInspectionMountsReadOnlyWithoutChown(t *testing.T) { + action := ports.RuntimeWorkerAction{Mode: ports.RuntimeWorkerModeInspect, RuntimeID: "inspect", Artifact: "registry.local/source:tag", MountPath: "/scroll"} + job := workerPullJobSpec("fixture", "inspect", "runtime-pvc", "druid:test", action, "", "", true, "druid-cli") + container := job.Spec.Template.Spec.Containers[0] + command := strings.Join(container.Command, " ") + if !strings.HasPrefix(command, "druid worker pull ") || strings.Contains(command, "chown") || !strings.Contains(command, "--mode inspect") { + t.Fatalf("unsafe inspection command: %s", command) + } + for _, mount := range container.VolumeMounts { + if !mount.ReadOnly { + t.Fatalf("inspection mount is writable: %#v", mount) + } + } +} + func TestSpawnPullWorkerCreateUsesFinalPVCAndWorkerJob(t *testing.T) { client := fake.NewSimpleClientset() backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, client) @@ -1628,11 +1646,14 @@ func TestDeleteRuntimePurgesServicesAndDataWhenRequested(t *testing.T) { } func TestBackupJobSpecUsesRuntimePVCAndRegistryEnv(t *testing.T) { - backup := backupJobSpec("druid", "backup", "runtime-pvc", "druid-cli:test", "registry.local/scroll:backup", "registry-secret", "", true) + backup := backupJobSpec("druid", "backup", "runtime-pvc", "druid-cli:test", "registry.local/scroll:backup", "registry-secret", "", true, true) assertFinishedJobTTL(t, backup) if backup.Spec.Template.Spec.Containers[0].Command[1] != "push" { t.Fatalf("backup command = %#v", backup.Spec.Template.Spec.Containers[0].Command) } + if !strings.Contains(strings.Join(backup.Spec.Template.Spec.Containers[0].Command, " "), "--preserve-release-manifest") { + t.Fatalf("backup command = %#v, want release manifest preservation", backup.Spec.Template.Spec.Containers[0].Command) + } if got := backup.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; got != "runtime-pvc" { t.Fatalf("backup PVC = %s, want runtime-pvc", got) } diff --git a/internal/runtime/kubernetes/state_store.go b/internal/runtime/kubernetes/state_store.go index 5be2403e..caff1fc9 100644 --- a/internal/runtime/kubernetes/state_store.go +++ b/internal/runtime/kubernetes/state_store.go @@ -19,21 +19,22 @@ import ( const ( runtimeStateComponent = "runtime-state" - configMapKeyID = "id" - configMapKeyOwnerID = "owner_id" - configMapKeyArtifact = "artifact" - configMapKeyArtifactDigest = "artifact_digest" - configMapKeyRoot = "root" - configMapKeyScrollName = "scroll_name" - configMapKeyScrollYAML = "scroll_yaml" - configMapKeyStatus = "status" - configMapKeyLastError = "last_error" - configMapKeyCreatedAt = "created_at" - configMapKeyUpdatedAt = "updated_at" - configMapKeyProceduresJSON = "procedures_json" - configMapKeyRoutingJSON = "routing_json" - configMapKeyReservedPorts = "reserved_ports_json" - configMapKeyUIPackagesJSON = "ui_packages_json" + configMapKeyID = "id" + configMapKeyOwnerID = "owner_id" + configMapKeyArtifact = "artifact" + configMapKeyArtifactDigest = "artifact_digest" + configMapKeyInstalledReleaseState = "installed_release_state" + configMapKeyRoot = "root" + configMapKeyScrollName = "scroll_name" + configMapKeyScrollYAML = "scroll_yaml" + configMapKeyStatus = "status" + configMapKeyLastError = "last_error" + configMapKeyCreatedAt = "created_at" + configMapKeyUpdatedAt = "updated_at" + configMapKeyProceduresJSON = "procedures_json" + configMapKeyRoutingJSON = "routing_json" + configMapKeyReservedPorts = "reserved_ports_json" + configMapKeyUIPackagesJSON = "ui_packages_json" ) type ConfigMapStateStore struct { @@ -205,21 +206,22 @@ func runtimeScrollConfigMap(namespace string, scroll *domain.RuntimeScroll) (*co }, }, Data: map[string]string{ - configMapKeyID: scroll.ID, - configMapKeyOwnerID: scroll.OwnerID, - configMapKeyArtifact: scroll.Artifact, - configMapKeyArtifactDigest: scroll.ArtifactDigest, - configMapKeyRoot: scroll.Root, - configMapKeyScrollName: scroll.ScrollName, - configMapKeyScrollYAML: scroll.ScrollYAML, - configMapKeyStatus: string(scroll.Status), - configMapKeyLastError: scroll.LastError, - configMapKeyCreatedAt: formatRuntimeTime(scroll.CreatedAt), - configMapKeyUpdatedAt: formatRuntimeTime(scroll.UpdatedAt), - configMapKeyProceduresJSON: string(procedures), - configMapKeyRoutingJSON: string(routing), - configMapKeyReservedPorts: string(reservedPorts), - configMapKeyUIPackagesJSON: string(uiPackages), + configMapKeyID: scroll.ID, + configMapKeyOwnerID: scroll.OwnerID, + configMapKeyArtifact: scroll.Artifact, + configMapKeyArtifactDigest: scroll.ArtifactDigest, + configMapKeyInstalledReleaseState: string(scroll.InstalledReleaseState), + configMapKeyRoot: scroll.Root, + configMapKeyScrollName: scroll.ScrollName, + configMapKeyScrollYAML: scroll.ScrollYAML, + configMapKeyStatus: string(scroll.Status), + configMapKeyLastError: scroll.LastError, + configMapKeyCreatedAt: formatRuntimeTime(scroll.CreatedAt), + configMapKeyUpdatedAt: formatRuntimeTime(scroll.UpdatedAt), + configMapKeyProceduresJSON: string(procedures), + configMapKeyRoutingJSON: string(routing), + configMapKeyReservedPorts: string(reservedPorts), + configMapKeyUIPackagesJSON: string(uiPackages), }, }, nil } @@ -263,21 +265,22 @@ func runtimeScrollFromConfigMap(configMap *corev1.ConfigMap) (*domain.RuntimeScr id = configMap.Labels[labelScrollID] } scroll := &domain.RuntimeScroll{ - ID: id, - OwnerID: data[configMapKeyOwnerID], - Artifact: data[configMapKeyArtifact], - ArtifactDigest: data[configMapKeyArtifactDigest], - Root: data[configMapKeyRoot], - ScrollName: data[configMapKeyScrollName], - ScrollYAML: data[configMapKeyScrollYAML], - Status: domain.RuntimeScrollStatus(data[configMapKeyStatus]), - LastError: data[configMapKeyLastError], - Routing: routing, - ReservedPorts: reservedPorts, - UIPackages: uiPackages, - CreatedAt: parseRuntimeTime(data[configMapKeyCreatedAt]), - UpdatedAt: parseRuntimeTime(data[configMapKeyUpdatedAt]), - Procedures: procedures, + ID: id, + OwnerID: data[configMapKeyOwnerID], + Artifact: data[configMapKeyArtifact], + ArtifactDigest: data[configMapKeyArtifactDigest], + InstalledReleaseState: domain.InstalledReleaseState(data[configMapKeyInstalledReleaseState]), + Root: data[configMapKeyRoot], + ScrollName: data[configMapKeyScrollName], + ScrollYAML: data[configMapKeyScrollYAML], + Status: domain.RuntimeScrollStatus(data[configMapKeyStatus]), + LastError: data[configMapKeyLastError], + Routing: routing, + ReservedPorts: reservedPorts, + UIPackages: uiPackages, + CreatedAt: parseRuntimeTime(data[configMapKeyCreatedAt]), + UpdatedAt: parseRuntimeTime(data[configMapKeyUpdatedAt]), + Procedures: procedures, } if scroll.Status == "" { scroll.Status = domain.RuntimeScrollStatusCreated diff --git a/internal/runtime/kubernetes/state_store_test.go b/internal/runtime/kubernetes/state_store_test.go index 0e4bc9c0..4a58828f 100644 --- a/internal/runtime/kubernetes/state_store_test.go +++ b/internal/runtime/kubernetes/state_store_test.go @@ -6,20 +6,33 @@ import ( "time" "github.com/highcard-dev/daemon/internal/core/domain" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) +func TestConfigMapStateStoreLegacyReleaseRemainsUnverified(t *testing.T) { + legacy := &corev1.ConfigMap{Data: map[string]string{ + configMapKeyID: "legacy", + configMapKeyArtifactDigest: "old-source-digest", + }} + scroll, err := runtimeScrollFromConfigMap(legacy) + if err != nil || scroll.InstalledReleaseState != "" || scroll.ArtifactDigest != "old-source-digest" { + t.Fatalf("legacy release = %#v, %v", scroll, err) + } +} + func TestConfigMapStateStoreRoundTripsRuntimeScroll(t *testing.T) { store := NewConfigMapStateStoreWithClient("druid", fake.NewSimpleClientset()) exitCode := 7 scroll := &domain.RuntimeScroll{ - ID: "container-lab", - Artifact: "registry.local/container-lab:1.0", - Root: ref("druid", "druid-container-lab-data"), - ScrollName: "container-lab", - ScrollYAML: "name: container-lab\n", - Status: domain.RuntimeScrollStatusCreated, + InstalledReleaseState: domain.InstalledReleasePending, + ID: "container-lab", + Artifact: "registry.local/container-lab:1.0", + Root: ref("druid", "druid-container-lab-data"), + ScrollName: "container-lab", + ScrollYAML: "name: container-lab\n", + Status: domain.RuntimeScrollStatusCreated, ReservedPorts: []domain.Port{ {Name: "vscode", Port: 3333, Protocol: "http"}, }, @@ -41,6 +54,10 @@ func TestConfigMapStateStoreRoundTripsRuntimeScroll(t *testing.T) { if got.Artifact != scroll.Artifact || got.Root != scroll.Root || got.ScrollYAML != scroll.ScrollYAML { t.Fatalf("stored scroll mismatch: %#v", got) } + if got.InstalledReleaseState != domain.InstalledReleasePending { + t.Fatalf("pending state lost: %q", got.InstalledReleaseState) + } + got.InstalledReleaseState = domain.InstalledReleaseReady if got.Procedures["verify"]["verify.0"].Status != domain.ScrollLockStatusError { t.Fatalf("procedure status = %s, want error", got.Procedures["verify"]["verify.0"].Status) } @@ -61,6 +78,9 @@ func TestConfigMapStateStoreRoundTripsRuntimeScroll(t *testing.T) { if got.ReservedPorts == nil || len(got.ReservedPorts) != 0 { t.Fatalf("reserved ports = %#v, want explicit empty list", got.ReservedPorts) } + if got.InstalledReleaseState != domain.InstalledReleaseReady { + t.Fatalf("confirmed state lost: %q", got.InstalledReleaseState) + } got.Status = domain.RuntimeScrollStatusRunning got.Procedures["verify"]["verify.0"] = domain.LockStatus{Status: domain.ScrollLockStatusDone, LastStatusChange: 456} diff --git a/internal/runtime/kubernetes/workers.go b/internal/runtime/kubernetes/workers.go index dbe8d68b..d90bd550 100644 --- a/internal/runtime/kubernetes/workers.go +++ b/internal/runtime/kubernetes/workers.go @@ -92,7 +92,7 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke return result, nil } -func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error { +func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error { if artifact == "" { return fmt.Errorf("backup artifact is required") } @@ -111,7 +111,7 @@ func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact strin return err } defer cleanupRegistryConfig() - job := backupJobSpec(namespace, jobName("backup", root, shortHash(artifact)), pvc, b.config.PullImage, artifact, b.config.RegistrySecret, registryConfigSecret, b.config.RegistryPlainHTTP) + job := backupJobSpec(namespace, jobName("backup", root, shortHash(artifact)), pvc, b.config.PullImage, artifact, b.config.RegistrySecret, registryConfigSecret, b.config.RegistryPlainHTTP, preserveReleaseManifest) if err := b.pinPodToRuntimeNode(ctx, namespace, pvc, &job.Spec.Template.Spec); err != nil { return err } diff --git a/internal/utils/copy.go b/internal/utils/copy.go new file mode 100644 index 00000000..0af91f05 --- /dev/null +++ b/internal/utils/copy.go @@ -0,0 +1,102 @@ +package utils + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" +) + +// CopyPath copies files, directories and symlinks without dereferencing links. +// Ordinary access bits are retained regardless of umask. preserveOwner additionally +// retains numeric UID/GID, failing if this process cannot set them. Callers must +// supply a trusted staging destination, not a live tree with symlink parents. +func CopyPath(src, dst string, preserveOwner bool) (err error) { + info, err := os.Lstat(src) + if err != nil { + return err + } + defer func() { + if err == nil && preserveOwner { + var header *tar.Header + header, err = tar.FileInfoHeader(info, "") + if err == nil { + err = SetPathOwner(dst, header.Uid, header.Gid) + } + } + }() + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + return os.Symlink(target, dst) + } + if info.IsDir() { + // Keep owner access while populating; finalize read-only directories last. + if err := os.MkdirAll(dst, 0700); err != nil { + return err + } + if err := os.Chmod(dst, info.Mode().Perm()|0700); err != nil { + return err + } + entries, err := os.ReadDir(src) + if err != nil { + return err + } + for _, entry := range entries { + if err := CopyPath(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name()), preserveOwner); err != nil { + return err + } + } + return os.Chmod(dst, info.Mode().Perm()) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("unsupported copy source type: %s", info.Mode().Type()) + } + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Chmod(info.Mode().Perm()) +} + +// SetPathOwner changes numeric ownership only when needed, never following a +// symlink. Source trees and external symlink targets must not be mutated. +func SetPathOwner(path string, uid, gid int) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + if uid == header.Uid && gid == header.Gid { + return nil + } + if err := os.Lchown(path, uid, gid); err != nil { + return fmt.Errorf("set materialized path owner: %w", err) + } + return nil +} diff --git a/internal/utils/fs.go b/internal/utils/fs.go index 994d09b5..71841671 100644 --- a/internal/utils/fs.go +++ b/internal/utils/fs.go @@ -178,7 +178,7 @@ func expandChunkNode(dataDir string, parentPath string, chunk *domain.Chunks) ([ return nil, fmt.Errorf("chunk %q has empty path", chunk.Name) } - paths, err := expandChunkPaths(dataDir, parentPath, chunk.Path) + paths, err := ExpandChunkPaths(dataDir, parentPath, chunk.Path) if err != nil { return nil, fmt.Errorf("failed to expand chunk %q path %q: %w", chunk.Name, chunk.Path, err) } @@ -218,7 +218,9 @@ func expandChunkNode(dataDir string, parentPath string, chunk *domain.Chunks) ([ return expanded, nil } -func expandChunkPaths(dataDir string, parentPath string, chunkPath string) ([]string, error) { +// ExpandChunkPaths resolves nested chunk paths and globs using the same rules +// for release packaging and installed-data update protection. +func ExpandChunkPaths(dataDir string, parentPath string, chunkPath string) ([]string, error) { resolvedPath, err := resolveChunkPath(parentPath, chunkPath) if err != nil { return nil, err diff --git a/test/integration/docker/docker_cli_test.go b/test/integration/docker/docker_cli_test.go index 71aeb739..28953047 100644 --- a/test/integration/docker/docker_cli_test.go +++ b/test/integration/docker/docker_cli_test.go @@ -147,7 +147,13 @@ func TestDockerBackendVolumeStorageWorkerLifecycleBackupRestore(t *testing.T) { } e2e.UnixJSONRequest(t, socket, http.MethodPost, "/api/v1/scrolls/"+created.ID+"/backup", fmt.Sprintf(`{"artifact":%q}`, backupArtifact)) + if got := e2e.WaitHTTP(t, fmt.Sprintf("http://127.0.0.1:%d/index.txt", fixture.RoutePort)); !strings.Contains(got, "healthy") { + t.Fatalf("backup restart index = %q, want healthy", got) + } writeDockerRootFile(t, volume, "data/public/index.txt", "mutated\n") + if got := readDockerRootFile(t, volume, "data/public/index.txt"); !strings.Contains(got, "mutated") { + t.Fatalf("mutated root index = %q, want mutated", got) + } if got := e2e.WaitHTTP(t, fmt.Sprintf("http://127.0.0.1:%d/index.txt", fixture.RoutePort)); !strings.Contains(got, "mutated") { t.Fatalf("mutated index = %q, want mutated", got) } diff --git a/test/integration/docker/permissions_test.go b/test/integration/docker/permissions_test.go new file mode 100644 index 00000000..d361803b --- /dev/null +++ b/test/integration/docker/permissions_test.go @@ -0,0 +1,109 @@ +//go:build integration && docker + +package docker_test + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/highcard-dev/daemon/internal/core/ports" + dockerruntime "github.com/highcard-dev/daemon/internal/runtime/docker" + "github.com/highcard-dev/daemon/test/integration/internal/e2e" +) + +func TestDockerUpdatePreservesProtectedOwner(t *testing.T) { + e2e.RequireDocker(t) + name := fmt.Sprintf("druid-e2e-docker-volume-owner-%d", time.Now().UnixNano()) + baseImage := e2e.BuildDockerImage(t, "druid-cli-e2e:"+name) + image := baseImage + "-nonroot" + buildCtx, buildCancel := context.WithTimeout(context.Background(), time.Minute) + defer buildCancel() + build := exec.CommandContext(buildCtx, "docker", "build", "-t", image, "-") + build.Stdin = strings.NewReader("FROM " + baseImage + "\nUSER 1000:1000\n") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("non-root worker fixture: %v: %s", err, output) + } + t.Cleanup(func() { e2e.Run(t, "docker", "image", "rm", image) }) + e2e.Run(t, "docker", "volume", "create", name) + root := "docker-volume://" + name + t.Cleanup(func() { + // The only volume removed is this test's uniquely named fixture. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if output, err := exec.CommandContext(ctx, "docker", "volume", "rm", name).CombinedOutput(); err != nil { + t.Logf("fixture volume cleanup: %v: %s", err, output) + } + }) + yaml := "name: owner-fixture\nchunks:\n - name: config\n path: private.cfg\n skip_update: true\n - name: saves\n path: saves\n skip_update: true\n" + candidate := t.TempDir() + if err := os.WriteFile(filepath.Join(candidate, "scroll.yaml"), []byte(yaml), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(candidate, "data"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(candidate, "data", "release.cfg"), []byte("new default"), 0600); err != nil { + t.Fatal(err) + } + e2e.Run(t, "docker", "run", "--rm", "--user", "0", "-v", name+":/scroll", "--entrypoint", "sh", image, "-c", "mkdir -p /scroll/data/saves; printf '%s' \"$1\" > /scroll/scroll.yaml; printf fixture > /scroll/data/private.cfg; chmod 600 /scroll/data/private.cfg; chown -R 1000:1000 /scroll; printf saved > /scroll/data/saves/save; chmod 700 /scroll/data/saves; chmod 600 /scroll/data/saves/save; ln -s /not-mounted /scroll/data/saves/link; chown -hR 1234:2345 /scroll/data/saves", "fixture", yaml) + backend, err := dockerruntime.NewWithConfig(dockerruntime.Config{WorkerImage: image}, ports.ProcedureStatusObserverFunc(func(ports.ProcedureStatusUpdate) {})) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + done, err := backend.SpawnPullWorker(ctx, ports.RuntimeWorkerAction{RuntimeID: name, RootRef: root, Artifact: candidate, Mode: ports.RuntimeWorkerModeUpdate}) + if err == nil { + err = <-done + } + if err != nil { + t.Fatal(err) + } + got := strings.TrimSpace(e2e.Run(t, "docker", "run", "--rm", "--user", "0", "-v", name+":/scroll:ro", "--entrypoint", "stat", image, "-c", "%a:%u:%g", "/scroll/data/private.cfg")) + if got != "600:1000:1000" { + t.Errorf("update changed protected metadata: %s", got) + } + // Check actual access as the original owner, not just metadata inspection. + content := e2e.Run(t, "docker", "run", "--rm", "--user", "1000:1000", "-v", name+":/scroll:ro", "--entrypoint", "cat", image, "/scroll/data/private.cfg") + if content != "fixture" { + t.Fatalf("protected content changed: %q", content) + } + for path, want := range map[string]string{"saves": "700:1234:2345", "saves/save": "600:1234:2345", "saves/link": "777:1234:2345", "release.cfg": "644:1000:1000"} { + got := strings.TrimSpace(e2e.Run(t, "docker", "run", "--rm", "--user", "0", "-v", name+":/scroll:ro", "--entrypoint", "stat", image, "-c", "%a:%u:%g", "/scroll/data/"+path)) + if got != want { + t.Errorf("%s metadata=%s, want %s", path, got, want) + } + } + if got := e2e.Run(t, "docker", "run", "--rm", "--user", "1234:2345", "-v", name+":/scroll", "--entrypoint", "sh", image, "-c", "cat /scroll/data/saves/save; printf writable > /scroll/data/saves/new"); got != "saved" { + t.Fatalf("custom-owner workload access: %q", got) + } + if got := e2e.Run(t, "docker", "run", "--rm", "--user", "1000:1000", "-v", name+":/scroll", "--entrypoint", "sh", image, "-c", "cat /scroll/data/release.cfg; printf writable > /scroll/data/new"); got != "new default" { + t.Fatalf("default-owner workload access: %q", got) + } + t.Run("owner-failure-keeps-installed-root", func(t *testing.T) { + if err := os.WriteFile(filepath.Join(candidate, "scroll.yaml"), []byte(strings.Replace(yaml, "owner-fixture", "rejected-fixture", 1)), 0644); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + // Deliberately remove chown capability only from this disposable worker. + // It must fail before swapping in a tree with incorrect protected owners. + out, err := exec.CommandContext(ctx, "docker", "run", "--rm", "--user", "0", "--cap-drop", "CHOWN", "-v", name+":/scroll", "-v", candidate+":/candidate:ro", "--entrypoint", "druid", image, "worker", "pull", "--mode", "update", "--root", "/scroll", "--runtime-id", name, "--artifact", "/candidate").CombinedOutput() + if err == nil || !strings.Contains(string(out), "set materialized path owner") { + t.Fatalf("expected ownership failure: %v %s", err, out) + } + got := e2e.Run(t, "docker", "run", "--rm", "--user", "1000:1000", "-v", name+":/scroll:ro", "--entrypoint", "cat", image, "/scroll/scroll.yaml") + if got != yaml { + t.Fatal("failed ownership preservation replaced installed YAML") + } + if got := e2e.Run(t, "docker", "run", "--rm", "--user", "1234:2345", "-v", name+":/scroll:ro", "--entrypoint", "cat", image, "/scroll/data/saves/new"); got != "writable" { + t.Fatal("failed ownership preservation changed installed data/access") + } + }) +} diff --git a/test/integration/internal/e2e/harness.go b/test/integration/internal/e2e/harness.go index 7c4f673e..4e15b91f 100644 --- a/test/integration/internal/e2e/harness.go +++ b/test/integration/internal/e2e/harness.go @@ -462,7 +462,7 @@ func DockerHostAddress(t *testing.T) string { return gateway } -func UnixJSONRequest(t *testing.T, socket string, method string, path string, body string) string { +func UnixJSONRequest(t *testing.T, socket string, method string, path string, body string, expectedStatus ...int) string { t.Helper() transport := &http.Transport{ DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { @@ -490,7 +490,7 @@ func UnixJSONRequest(t *testing.T, socket string, method string, path string, bo if err != nil { t.Fatal(err) } - if resp.StatusCode >= 400 { + if (len(expectedStatus) == 0 && resp.StatusCode >= 400) || (len(expectedStatus) > 0 && resp.StatusCode != expectedStatus[0]) { t.Fatalf("%s %s failed with %d: %s", method, path, resp.StatusCode, data) } return string(data) diff --git a/test/integration/kubernetes/kubernetes_cli_test.go b/test/integration/kubernetes/kubernetes_cli_test.go index 182b20e1..31fbaba3 100644 --- a/test/integration/kubernetes/kubernetes_cli_test.go +++ b/test/integration/kubernetes/kubernetes_cli_test.go @@ -4,6 +4,7 @@ package kubernetes_test import ( "context" + "encoding/json" "fmt" "net/http" "os" @@ -29,6 +30,25 @@ func TestKubernetesBackendCLIComplexLifecycle(t *testing.T) { namespace := "druid-cli-e2e-" + suffix name := "k8s-cli-" + suffix fixture := e2e.WriteFixture(t, filepath.Join(t.TempDir(), "scroll"), name, port, routePort) + fixtureYAML, err := os.ReadFile(filepath.Join(fixture.Dir, "scroll.yaml")) + if err != nil { + t.Fatal(err) + } + fixtureYAML = append(fixtureYAML, []byte("\nchunks:\n - name: finite\n path: finite.txt\n skip_update: true\n - name: version\n path: version.txt\n")...) + if err := os.WriteFile(filepath.Join(fixture.Dir, "scroll.yaml"), fixtureYAML, 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(fixture.Dir, "data"), 0755); err != nil { + t.Fatal(err) + } + writeFixtureData := func(file, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(fixture.Dir, "data", file), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + writeFixtureData("finite.txt", "initial") + writeFixtureData("version.txt", "v1") workerImage := e2e.BuildDockerImage(t, "druid-cli-e2e:"+name) importImageIntoCurrentCluster(t, workerImage) pushArtifact := fmt.Sprintf("127.0.0.1:%d/druid-e2e/%s:v1", registryPort, name) @@ -120,6 +140,58 @@ func TestKubernetesBackendCLIComplexLifecycle(t *testing.T) { t.Fatalf("stopped status = %s, want stopped", stopped.Status) } waitKubernetesResourcesGone(t, namespace, pvc, "statefulset,job,pod") + + // Exercise actual changed workers against an isolated PVC/registry, without + // consuming any canonical local ReservedPorts or touching user deployments. + releasePath := "/api/v1/scrolls/" + created.ID + "/release" + readRelease := func() map[string]string { + t.Helper() + var release map[string]string + if err := json.Unmarshal([]byte(e2e.UnixJSONRequest(t, socket, http.MethodGet, releasePath, "")), &release); err != nil { + t.Fatal(err) + } + return release + } + originalRelease := readRelease() + originalManifest := readPVCFile(t, namespace, pvc, "manifest.json") + backupArtifact := strings.TrimSuffix(runtimeArtifact, ":v1") + "-backup:v1" + e2e.UnixJSONRequest(t, socket, http.MethodPost, "/api/v1/scrolls/"+created.ID+"/backup", fmt.Sprintf(`{"artifact":%q}`, backupArtifact)) + writeFixtureData("version.txt", "v2") + writeFixtureData("finite.txt", "upstream replacement must not win") + e2e.RunEnv(t, []string{"DRUID_REGISTRY_PLAIN_HTTP=true"}, bins.Druid, "push", pushArtifact, fixture.Dir) + acceptedDigest := resolveRegistryCandidate(t, fmt.Sprintf("http://127.0.0.1:%d/v2/druid-e2e/%s/manifests/v1", registryPort, name)) + // Move the tag again after acceptance: update must still install v2. + writeFixtureData("version.txt", "v3-unaccepted") + e2e.RunEnv(t, []string{"DRUID_REGISTRY_PLAIN_HTTP=true"}, bins.Druid, "push", pushArtifact, fixture.Dir) + acceptedArtifact := strings.TrimSuffix(runtimeArtifact, ":v1") + "@" + acceptedDigest + updatePath := "/api/v1/scrolls/" + created.ID + "/update" + acceptedBody := fmt.Sprintf(`{"artifact":%q,"expected_installed_digest":%q}`, acceptedArtifact, originalRelease["digest"]) + e2e.UnixJSONRequest(t, socket, http.MethodPost, updatePath, acceptedBody) + // A concurrent restore/update invalidates an earlier acceptance. The + // precondition is checked against the actual PVC before any mutation. + e2e.UnixJSONRequest(t, socket, http.MethodPost, updatePath, acceptedBody, http.StatusConflict) + if got := readPVCFile(t, namespace, pvc, "data/version.txt"); strings.TrimSpace(got) != "v2" { + t.Fatalf("update followed moved tag: %q", got) + } + if got := readPVCFile(t, namespace, pvc, "data/finite.txt"); !strings.Contains(got, "finite-ok") { + t.Fatalf("update overwrote protected state: %q", got) + } + if got := readRelease(); got["digest"] != acceptedDigest { + t.Fatalf("installed release=%v, want %s", got, acceptedDigest) + } + e2e.UnixJSONRequest(t, socket, http.MethodPost, "/api/v1/scrolls/"+created.ID+"/restore", fmt.Sprintf(`{"artifact":%q,"restart":false}`, backupArtifact)) + if got := readPVCFile(t, namespace, pvc, "data/version.txt"); strings.TrimSpace(got) != "v1" { + t.Fatalf("restore contents=%q", got) + } + if got := readPVCFile(t, namespace, pvc, "data/record-env.txt"); !strings.Contains(got, "USER_ENV=finite") { + t.Fatalf("backup omitted runtime-created data outside release chunks: %q", got) + } + if got := readPVCFile(t, namespace, pvc, "manifest.json"); got != originalManifest { + t.Fatal("restore changed original installed descriptor bytes") + } + if got := readRelease(); got["digest"] != originalRelease["digest"] { + t.Fatalf("restored release=%v, want %v", got, originalRelease) + } deleted := e2e.RunClient(t, bins, socket, "delete", created.ID) if !strings.Contains(deleted, `"status": "deleted"`) { t.Fatalf("delete response = %s, want deleted status", deleted) diff --git a/test/integration/kubernetes/registry_resolution_test.go b/test/integration/kubernetes/registry_resolution_test.go new file mode 100644 index 00000000..1a2f0511 --- /dev/null +++ b/test/integration/kubernetes/registry_resolution_test.go @@ -0,0 +1,44 @@ +//go:build integration && kubernetes + +package kubernetes_test + +import ( + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/highcard-dev/daemon/test/integration/internal/e2e" +) + +func TestRegistryCandidateResolution(t *testing.T) { + bins := e2e.BuildBinaries(t) + port := e2e.StartRegistry(t) + fixture := e2e.WriteFixture(t, filepath.Join(t.TempDir(), "scroll"), "registry-resolution", 8080, 8081) + artifact := fmt.Sprintf("127.0.0.1:%d/druid-e2e/registry-resolution:v1", port) + e2e.RunEnv(t, []string{"DRUID_REGISTRY_PLAIN_HTTP=true"}, bins.Druid, "push", artifact, fixture.Dir) + resolveRegistryCandidate(t, fmt.Sprintf("http://127.0.0.1:%d/v2/druid-e2e/registry-resolution/manifests/v1", port)) +} + +func resolveRegistryCandidate(t *testing.T, url string) string { + t.Helper() + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Accept", "application/vnd.oci.image.manifest.v1+json") + response, err := (&http.Client{Timeout: 10 * time.Second}).Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(io.LimitReader(response.Body, 1024)) + digest := response.Header.Get("Docker-Content-Digest") + if response.StatusCode != http.StatusOK || !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("candidate lookup: status=%d body=%s", response.StatusCode, body) + } + return digest +} diff --git a/test/mock/services.go b/test/mock/services.go index cd3883b0..90fb284a 100644 --- a/test/mock/services.go +++ b/test/mock/services.go @@ -211,17 +211,17 @@ func (m *MockRuntimeBackendInterface) EXPECT() *MockRuntimeBackendInterfaceMockR } // BackupRuntime mocks base method. -func (m *MockRuntimeBackendInterface) BackupRuntime(ctx context.Context, root, artifact string, registryCredentials []domain.RegistryCredential) error { +func (m *MockRuntimeBackendInterface) BackupRuntime(ctx context.Context, root, artifact string, registryCredentials []domain.RegistryCredential, preserveReleaseManifest bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BackupRuntime", ctx, root, artifact, registryCredentials) + ret := m.ctrl.Call(m, "BackupRuntime", ctx, root, artifact, registryCredentials, preserveReleaseManifest) ret0, _ := ret[0].(error) return ret0 } // BackupRuntime indicates an expected call of BackupRuntime. -func (mr *MockRuntimeBackendInterfaceMockRecorder) BackupRuntime(ctx, root, artifact, registryCredentials any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) BackupRuntime(ctx, root, artifact, registryCredentials, preserveReleaseManifest any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackupRuntime", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).BackupRuntime), ctx, root, artifact, registryCredentials) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackupRuntime", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).BackupRuntime), ctx, root, artifact, registryCredentials, preserveReleaseManifest) } // CreateUIPackageUpload mocks base method. From 70a6128767529718d3ca41c72cdab3a46eb15c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Schottst=C3=A4dt?= Date: Tue, 27 Oct 2026 02:12:49 +0100 Subject: [PATCH 2/2] refactor(runtime)!: remove release inspection Use only the installed release recorded by a successful pull. Remove the inspect worker mode and its Docker and Kubernetes special cases. BREAKING CHANGE: older runtimes without confirmed release metadata return an error instead of starting a compatibility inspection worker. --- api/openapi.yaml | 2 +- apps/druid/adapters/cli/worker_pull.go | 15 ++- .../druid/adapters/cli/worker_release_test.go | 28 +++-- apps/druid/core/services/runtime_release.go | 54 +-------- .../core/services/runtime_release_test.go | 74 ++++-------- .../services/runtime_restore_failure_test.go | 3 +- apps/druid/core/services/runtime_update.go | 2 +- internal/api/generated.go | 108 +++++++++--------- internal/core/domain/runtime_scroll.go | 5 +- internal/core/ports/services_ports.go | 1 - internal/runtime/docker/workers.go | 4 +- internal/runtime/kubernetes/resources.go | 9 -- internal/runtime/kubernetes/resources_test.go | 15 --- 13 files changed, 114 insertions(+), 206 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index 594134ee..74e23abc 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -492,7 +492,7 @@ paths: get: operationId: getInstalledRelease tags: [scrolls] - summary: Read the installed release descriptor from the runtime volume + summary: Read the installed release descriptor recorded by the last successful pull parameters: - name: id in: path diff --git a/apps/druid/adapters/cli/worker_pull.go b/apps/druid/adapters/cli/worker_pull.go index 4dd382cc..65ebc5fa 100644 --- a/apps/druid/adapters/cli/worker_pull.go +++ b/apps/druid/adapters/cli/worker_pull.go @@ -57,13 +57,19 @@ func init() { // Older daemons pass this flag; restore mode always preserves the manifest. WorkerPullCommand.Flags().Bool("preserve-release-manifest", false, "Deprecated compatibility flag; restore always preserves the release manifest") WorkerPullCommand.Flags().MarkHidden("preserve-release-manifest") - WorkerPullCommand.Flags().StringVar(&workerPullMode, "mode", string(ports.RuntimeWorkerModeCreate), "Worker mode: create, update, restore, or inspect") + WorkerPullCommand.Flags().StringVar(&workerPullMode, "mode", string(ports.RuntimeWorkerModeCreate), "Worker mode: create, update, or restore") WorkerPullCommand.MarkFlagRequired("artifact") WorkerPullCommand.MarkFlagRequired("runtime-id") } func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { result := ports.RuntimeWorkerResult{} + switch action.Mode { + case ports.RuntimeWorkerModeCreate, ports.RuntimeWorkerModeUpdate, ports.RuntimeWorkerModeRestore: + default: + result.Error = fmt.Sprintf("unsupported worker pull mode %q", action.Mode) + return result + } if action.Artifact == "" { result.Error = "artifact is required" return result @@ -72,9 +78,6 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { if root == "" { root = "/scroll" } - if action.Mode == ports.RuntimeWorkerModeInspect { - return inspectInstalledRelease(root) - } owner, err := resolveWorkerDataOwner(action.NewDataOwner) if err != nil { result.Error = err.Error() @@ -99,10 +102,6 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult { return readInstalledRelease(root, localErr != nil || action.Mode == ports.RuntimeWorkerModeRestore) } -func inspectInstalledRelease(root string) ports.RuntimeWorkerResult { - return readInstalledRelease(root, true) -} - func readInstalledRelease(root string, requireDescriptor bool) ports.RuntimeWorkerResult { result := ports.RuntimeWorkerResult{} scrollYAML, err := os.ReadFile(filepath.Join(root, "scroll.yaml")) diff --git a/apps/druid/adapters/cli/worker_release_test.go b/apps/druid/adapters/cli/worker_release_test.go index 29641eef..39158469 100644 --- a/apps/druid/adapters/cli/worker_release_test.go +++ b/apps/druid/adapters/cli/worker_release_test.go @@ -9,20 +9,24 @@ import ( "github.com/highcard-dev/daemon/internal/core/ports" ) -func TestInspectWorkerReadsInstalledReleaseNotRegistryReference(t *testing.T) { - root := t.TempDir() - want := "sha256:" + strings.Repeat("a", 64) - raw := "{\n \"digest\": \"" + want + "\"\n}" - mustWrite(t, filepath.Join(root, "manifest.json"), raw) - mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: registry.local/owned/example\napp_version: '1'\ncommands: {}\n") - result := runWorkerPull(ports.RuntimeWorkerAction{Mode: ports.RuntimeWorkerModeInspect, MountPath: root, Artifact: "unreachable.invalid/backup:latest"}) - if result.Error != "" || result.ArtifactDigest != want { - t.Fatalf("inspection = %#v", result) +func TestPullWorkerRejectsUnsupportedModesWithoutChangingRuntime(t *testing.T) { + for _, mode := range []ports.RuntimeWorkerMode{"inspect", "unknown"} { + t.Run(string(mode), func(t *testing.T) { + source, root := t.TempDir(), t.TempDir() + mustWrite(t, filepath.Join(source, "scroll.yaml"), "name: replacement\n") + mustWrite(t, filepath.Join(root, "scroll.yaml"), "name: installed\n") + mustWrite(t, filepath.Join(root, "manifest.json"), "installed descriptor") + result := runWorkerPull(ports.RuntimeWorkerAction{Mode: mode, MountPath: root, Artifact: source}) + if !strings.Contains(result.Error, "unsupported worker pull mode") || result.ScrollYAML != "" || result.ArtifactDigest != "" { + t.Fatalf("unsupported mode result = %#v", result) + } + assertFile(t, filepath.Join(root, "scroll.yaml"), "name: installed\n") + assertFile(t, filepath.Join(root, "manifest.json"), "installed descriptor") + }) } - assertFile(t, filepath.Join(root, "manifest.json"), raw) } -func TestInspectWorkerRejectsMissingOrInvalidInstalledDescriptor(t *testing.T) { +func TestReadInstalledReleaseRejectsMissingOrInvalidDescriptor(t *testing.T) { for _, raw := range []string{"", "{}", `{"digest":"sha256:bad"}`} { t.Run(raw, func(t *testing.T) { root := t.TempDir() @@ -30,7 +34,7 @@ func TestInspectWorkerRejectsMissingOrInvalidInstalledDescriptor(t *testing.T) { if raw != "" { mustWrite(t, filepath.Join(root, "manifest.json"), raw) } - if result := inspectInstalledRelease(root); result.Error == "" { + if result := readInstalledRelease(root, true); result.Error == "" { t.Fatal("invalid installed release was accepted") } }) diff --git a/apps/druid/core/services/runtime_release.go b/apps/druid/core/services/runtime_release.go index dd957c42..7ea90144 100644 --- a/apps/druid/core/services/runtime_release.go +++ b/apps/druid/core/services/runtime_release.go @@ -5,8 +5,6 @@ import ( "fmt" "github.com/highcard-dev/daemon/internal/core/domain" - "github.com/highcard-dev/daemon/internal/core/ports" - coreservices "github.com/highcard-dev/daemon/internal/core/services" ) // InstalledRelease serves the descriptor reported by the last successful pull. @@ -18,56 +16,14 @@ func (s *RuntimeSupervisor) InstalledRelease(ctx context.Context, id string) (ma if err != nil { return nil, err } - return s.installedRelease(ctx, runtime) + return s.installedRelease(runtime) } -func (s *RuntimeSupervisor) installedRelease(ctx context.Context, runtime *domain.RuntimeScroll) (map[string]string, error) { - switch runtime.InstalledReleaseState { - case domain.InstalledReleaseReady: - return installedReleaseIdentity(runtime.Root, runtime.ScrollYAML, runtime.ArtifactDigest) - case "": // Older records need one read of the actual installed descriptor. - default: - // A lost callback may leave a worker still replacing files. Do not - // inspect concurrently or advertise the previous cached release. - return nil, fmt.Errorf("installed release is unconfirmed after an incomplete materialization") +func (s *RuntimeSupervisor) installedRelease(runtime *domain.RuntimeScroll) (map[string]string, error) { + if runtime.InstalledReleaseState != domain.InstalledReleaseReady { + return nil, fmt.Errorf("installed release is unconfirmed; a successful pull is required") } - installed, err := s.runPullWorker(ctx, s.runtimeBackend, ports.RuntimeWorkerModeInspect, runtime.ID, runtime.Artifact, runtime.Root, nil, "") - if err != nil { - return nil, err - } - identity, err := installedReleaseIdentity(runtime.Root, string(installed.ScrollYAML), installed.ArtifactDigest) - if err != nil { - return nil, err - } - // Keep the live session in sync so subsequent status writes cannot replace - // the newly confirmed identity with its old copy. - s.mu.Lock() - session := s.sessions[runtime.ID] - s.mu.Unlock() - current := runtime - if session != nil { - session.mu.Lock() - defer session.mu.Unlock() - current = session.runtimeScroll - } - updated := *current - updated.ArtifactDigest = installed.ArtifactDigest - updated.ScrollYAML = string(installed.ScrollYAML) - updated.InstalledReleaseState = domain.InstalledReleaseReady - scrollService, err := coreservices.NewCachedScrollServiceWithPorts(updated.Root, installed.ScrollYAML, updated.ReservedPorts) - if err != nil { - return nil, err - } - updated.ScrollName = scrollService.GetCurrent().Name - if err := s.store.UpdateScroll(&updated); err != nil { - return nil, err - } - *runtime = updated - if session != nil { - *session.runtimeScroll = updated - session.scrollService = scrollService - } - return identity, nil + return installedReleaseIdentity(runtime.Root, runtime.ScrollYAML, runtime.ArtifactDigest) } func installedReleaseIdentity(root, yaml, digest string) (map[string]string, error) { diff --git a/apps/druid/core/services/runtime_release_test.go b/apps/druid/core/services/runtime_release_test.go index 0f9057dc..2a029e9f 100644 --- a/apps/druid/core/services/runtime_release_test.go +++ b/apps/druid/core/services/runtime_release_test.go @@ -57,31 +57,30 @@ func TestMaterializationCachesInstalledReleaseAcrossRestart(t *testing.T) { } } -func TestLegacyInstalledReleaseInspectedOnceAndSessionUpdated(t *testing.T) { - store := newTestStateStore(t) - legacy := &domain.RuntimeScroll{ID: "legacy", Root: "runtime://legacy", Artifact: "registry.local/backup:v1", ArtifactDigest: "sha256:" + strings.Repeat("b", 64), ScrollYAML: cachedScrollYAML(""), Status: domain.RuntimeScrollStatusStopped} - if err := store.CreateScroll(legacy); err != nil { - t.Fatal(err) - } - callbacks := NewWorkerCallbackManager() - digest := "sha256:" + strings.Repeat("a", 64) - backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: releaseScrollYAML(), digest: digest} - supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) - supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") - session, err := supervisor.sessionFor(legacy.ID) - if err != nil { - t.Fatal(err) - } - identity, err := supervisor.InstalledRelease(context.Background(), legacy.ID) - if err != nil || identity["digest"] != digest || backend.action.Mode != ports.RuntimeWorkerModeInspect { - t.Fatalf("legacy repair = %v, %v", identity, err) - } - // A normal session write must not resurrect its legacy identity. - session.markError(errors.New("unrelated command error")) - restarted := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) - identity, err = restarted.InstalledRelease(context.Background(), legacy.ID) - if err != nil || identity["digest"] != digest || backend.spawnCount != 1 { - t.Fatalf("persisted repair = %v, %v; workers = %d", identity, err, backend.spawnCount) +func TestInstalledReleaseRejectsUnconfirmedStateWithoutWorker(t *testing.T) { + for _, state := range []domain.InstalledReleaseState{"", domain.InstalledReleasePending, "unknown"} { + t.Run(string(state), func(t *testing.T) { + store := newTestStateStore(t) + runtime := &domain.RuntimeScroll{ID: "unconfirmed", Root: "runtime://unconfirmed", Artifact: "registry.local/backup:v1", ArtifactDigest: "sha256:" + strings.Repeat("a", 64), ScrollYAML: releaseScrollYAML(), InstalledReleaseState: state, Status: domain.RuntimeScrollStatusStopped} + if err := store.CreateScroll(runtime); err != nil { + t.Fatal(err) + } + backend := &fakeWorkerBackend{} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + for i := 0; i < 2; i++ { + identity, err := supervisor.InstalledRelease(context.Background(), runtime.ID) + if err == nil || !strings.Contains(err.Error(), "unconfirmed") || identity != nil { + t.Fatalf("unconfirmed release query = %v, %v", identity, err) + } + } + if backend.spawnCount != 0 { + t.Fatal("unconfirmed release query spawned a worker") + } + stored, err := store.GetScroll(runtime.ID) + if err != nil || stored.InstalledReleaseState != state || stored.ArtifactDigest != runtime.ArtifactDigest { + t.Fatalf("query changed the recorded release: %#v, %v", stored, err) + } + }) } } @@ -98,7 +97,7 @@ func TestInstalledReleaseRejectsInvalidConfirmedIdentityWithoutWorker(t *testing t.Fatalf("invalid release query = %v", err) } if backend.spawnCount != 0 { - t.Fatal("invalid confirmed identity triggered inspection") + t.Fatal("invalid confirmed identity spawned a worker") } }) } @@ -147,29 +146,6 @@ func TestMaterializationRequiresDurableInvalidation(t *testing.T) { } } -func TestLegacyInspectionDoesNotConfirmCacheWhenPersistenceFails(t *testing.T) { - store := &failReleaseStateStore{RuntimeScrollStore: newTestStateStore(t), state: domain.InstalledReleaseReady} - if err := store.CreateScroll(&domain.RuntimeScroll{ID: "legacy", Artifact: "backup:v1", Root: "runtime://legacy", ScrollYAML: cachedScrollYAML("")}); err != nil { - t.Fatal(err) - } - callbacks := NewWorkerCallbackManager() - backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: releaseScrollYAML(), digest: "sha256:" + strings.Repeat("a", 64)} - supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) - supervisor.SetWorkerCallbacks(callbacks, "http://worker-callback") - session, err := supervisor.sessionFor("legacy") - if err != nil { - t.Fatal(err) - } - if _, err := supervisor.InstalledRelease(context.Background(), "legacy"); err == nil { - t.Fatal("inspection ignored persistence failure") - } - session.mu.Lock() - defer session.mu.Unlock() - if session.runtimeScroll.InstalledReleaseState != "" || session.runtimeScroll.ArtifactDigest != "" { - t.Fatal("failed write confirmed the in-memory cache") - } -} - func releaseScrollYAML() string { return strings.Replace(cachedScrollYAML(""), "name: cached", "name: registry.local/release", 1) } diff --git a/apps/druid/core/services/runtime_restore_failure_test.go b/apps/druid/core/services/runtime_restore_failure_test.go index d9333903..48ed8fc8 100644 --- a/apps/druid/core/services/runtime_restore_failure_test.go +++ b/apps/druid/core/services/runtime_restore_failure_test.go @@ -55,8 +55,7 @@ func TestRestoreIndeterminateFailureDoesNotRestart(t *testing.T) { if stored.InstalledReleaseState != domain.InstalledReleasePending { t.Fatalf("uncertain release was confirmed: %s", stored.InstalledReleaseState) } - // A daemon restart must not serve the old digest or inspect a volume - // which an orphaned worker could still be modifying. + // A daemon restart must not serve the old digest after a worker failure. restarted := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) if _, err := restarted.InstalledRelease(context.Background(), runtime.ID); err == nil || !strings.Contains(err.Error(), "unconfirmed") { t.Fatalf("uncertain release query = %v", err) diff --git a/apps/druid/core/services/runtime_update.go b/apps/druid/core/services/runtime_update.go index 4430eb9f..e0bf5ddd 100644 --- a/apps/druid/core/services/runtime_update.go +++ b/apps/druid/core/services/runtime_update.go @@ -30,7 +30,7 @@ func (s *RuntimeSupervisor) Update(id string, artifact string, registryCredentia if !acceptedUpdateReference.MatchString("expected@" + *expectedDigest) { return nil, ErrUnacceptedUpdate } - installed, err := s.installedRelease(context.Background(), runtimeScroll) + installed, err := s.installedRelease(runtimeScroll) if err != nil { return nil, err } diff --git a/internal/api/generated.go b/internal/api/generated.go index 7d08a312..a733dd11 100644 --- a/internal/api/generated.go +++ b/internal/api/generated.go @@ -2993,7 +2993,7 @@ type ServerInterface interface { // Get runtime queue state // (GET /api/v1/scrolls/{id}/queue) GetScrollQueue(c *fiber.Ctx, id string) error - // Read the installed release descriptor from the runtime volume + // Read the installed release descriptor recorded by the last successful pull // (GET /api/v1/scrolls/{id}/release) GetInstalledRelease(c *fiber.Ctx, id string) error // Execute runtime restore @@ -3425,59 +3425,59 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xbW3PbtrP/Khicvh1ZctokM/V5qev04jaZ+NjJ5CFxNRCwklCBAAKAtvX36Lv/BxdS", - "FAnKluK0dqcvbSwAi8VvF3vD8hZTVWglQTqLj26xpXMoSPjnsdZiea5Kx+XsHD6XYJ3/WRulwTgOYRKx", - "ls9kUS3nDorwj28MTPER/p/Rmvwo0R6dl9LxAjxpOK7X49UAu6UGfISJMWSJV6sBNvC55AYYPvq4sdVl", - "PVdN/gQaFp8YIA4uqFFC9PNrHJ8SGkYYWGq4dlxJfITfnpyiahQZmIIBSQEpg4SiRCAbCCNN3BwPMNyQ", - "QovIbFxjh8yUnA1ns5ED68J/jvx/cM2rdYbLmeeVsy4Dr0AboMQBQ0RwYtFUGSRJAUP0NszxTDgyEYBM", - "RBBxNooTTqdIFdw5YAPk5oAYgUJJNAMJhjiwiEjE2XCD8T/VxOZ48xQz8DwMC/8Xx7jVgizD6ZB1XAhE", - "VQEWTY0qEtLDJSnE/Tm2mtAM27+XEzAS/P71rABsxb8Bq0pDwQ7R6UwqAwxNlkgqedBYOiF0AZLZYW53", - "dS3BjHMSTYqOwgzEGSotsLA7La1TBZiDKaFczpDxdwGR0s2V4f8hfn12LwMzbp1ZjqkBBtJxIna4d2nx", - "Sb327jtXXZfchXsFAhyweOO6Vy0i0jmCdcSVYcJasCxS6p64xQ73UxKBHEc/SVuaXUxAh7tqcMz4LK3u", - "uby99+Zf9Xwc6vkrEOHm52C1kha6elAolpHISWkMSIfmYTWKyobC3KYpUovc+bVRMwPWdsmepRGkwVCQ", - "jsyinIUizCPsGQu4egM3VaYgDh/hqVDE+4+C3PCiLPDRs8PDAS64jH8d1izIspiASdfLuDEjLnO2D3OQ", - "Tdsc5oZrV+/oFx54rcADLEshvK3HR86UcNfdDBDl5PBa0cVFfek3ZQA33I1pEkTPflw6mMXDCWLdOIpk", - "TOdEzsK6mnku3cvnOLewYXSkR+4jNqWU/hgDzJQMsjVGGTzA14T7iKdxlJ4DJ5pZrnI4nCmTsUYbEtrF", - "quhErtaNly9efPeioR3PckBoo5yiSjShmDun8SD8L7hX6v8qmb4bgsBcYqVBO3t6oygwb50DUG+IDraY", - "MR7jirNNG93z+zb70dCzVYaBLkflRHA7f396RuiCzKDXYYSQb0tAFNzNgVHKHRgQxPErQMNrYosQLA7R", - "K5iSUjiLnELa8CviYMS4dSOidZynDNKeG7r5+zDrEDsHyRjOzhnmqseZaWLttTJ5l1ZaMD0K2NKEQL+x", - "oEE4pw3J9Rwn+/22sn77Oe0+t8Mi8PhoSoSFwcO5Ib9lMJ4NdiZKCSByNx+VcPCmoc9ETlQpWW6fQQB9", - "zHUWkzBW2YiuHVgA6GPBr+CdIdMpp1kawbAR6vgVd8sxcRvGtukpdrdaWcsUDUR+XcNudeV/M54sXYTr", - "Ps4gRFRZSq6DRgPuNLjTXtUatdhO85pLpq7zPO1yuh4DXWPbNdaDpGHrw9cIbdHYdvKe8ezOmwKxTT93", - "d3jj/tFtChKN65brUBqRt3Hbzs/l7B0xM8ic/n65wC63467D73t3LAigTpltXrerkm1ULJgrTmF8P2fR", - "o5XjnnCiRX6LVvalolu9Bw11I7aTfetJAYPBjIFkbriZivXL8E4PlQmlokcCcwUsaPn9s64QlYblhL2V", - "YtkKvtcOT6ke5xtvwoNX/wY4Blb9Wt8N6pMo8aAR3luntA6/VRF+VW24zAi25GMdw8H7HqSOH0PYWWq2", - "ozLlShy1vibcN7EYrFOPhu5u7L3ljtT89ge6XVR2PtUWk9o8rZ80wKmmuiP/eycKHSByFu19YGbvavJP", - "N1pwyp1YIkIpaAcM8aIoY/HUBbcxRG+4tSH7N2g9NFtXny0iBpABzxQwNIGpMoCCRvtlPpVPhaJhhNH7", - "XHyE//j4xw+fPtnL//3Bzsm3L14efSQH08OD7y9vXz5ffZOTFtzosMeYS+uIEMAaFbCexEcboEpG/BGd", - "A10AQ2RGPInAG6GuJALVJFFFSBlUSgYmzCqI93OSSApIKLpoHeW+J3hEtafgEGlpuFte+A1SIA/EgDku", - "4x2Lf/1cXaXfPrwLlqkJ9G8f3iGnFiBjWZwHztwSaaOuOAMTzIIn70PKQG6NS8jp/RH8+mrPTfIXc2Xc", - "gc8BGPpcgllWmymDPsDkQtEFOESVlECryhT3C8NkXEVqcYv1zkTz38HD5d2knCq/MVXSxWuyah/ylSk5", - "QyevT5EgpaTz8FDAUEGkv+IorOQSzEEocrLqGYZof79ixWyABF/AJzkLrwneCRo7QIw4MiEW7CAQvIZJ", - "NTb8FNjlLlTyagbwAPvRyNbh8NnwMPhsDZJojo/wd+GnaK2CQEdE89HVs1EsFfpfUiy4ecJfwFUFt42i", - "Ig7EY957yuLEWLMM8goePZQuw2bfHh5WSKZ4uwHB6E8by0dRn+/S9lZlNIhqk+c4I3jhF4ff/YUbX8RI", - "D5WSXBEey4HhPpVFQcwywdnG0ZGZDVWIKIkBjnjjS7+0ElPUHNuQ0yb8r7l1F2nOF4K/SyCUYtauuelg", - "U9Xrq4Ns4uLZr58NbH2OCpo00sTGR9o2A0TzIRVHowfW/ajY8sEUIfdWu9q0sD4MXXXk8OzBWGjBfxfc", - "qIotN1GPB2nhvh32rkqOILxbhfgiK5Hmu9ZXkkju6exeEjn82yQSUWtLJB6kJREEN9y66FpUil/EMj6A", - "2J3FdcvZKtp5n0h0xRUfRmtxaWJIAQ6M3+I2+tAU8iYXGoL+TaAHDdDaMfTlVxTC5qPu3UKokqnVAD8/", - "fN7/yJimS+XQNNSbNqUWt93pHg3yZvwXcE8T+R3V/0sR9370C82WvwcjH5eVut92/RjGv7JIHt4e3vVC", - "8dhsY4QZ+aXpQm5axRugZeOCJantJXGqioJIZke36V+rfumflzLyfBKnfg0NGGSJ0HrDnSi1XswJdyEh", - "Cql2FD0wlGgjp2rAq+Q8agDjcjbsyZfsUlLc5KL9TNV5UPpbrU4shLC2rfhC63NeyraLXgtsL52UUz7r", - "je1rp3AS5z1C19CuIXQEcUaMXSfA6cBdm65z0/bF1CoB9l6oxpmPENd8hXCjyN2PeXUwtIBlbL1aP1p0", - "oU+dAJYqHYxEDcoe4NdV/O3In6kYwD462HdJgRvP3zunwcgDVVUCHjwk2qC+lxw/l1DC3XL8/zDtiUWs", - "uYeorrzC0QKGsAXvz41Za6A/J1gqwPtxNiCA2K1In1Z16PM09xHC3WoO629CNaCV5U6Z5d2vLI25g4rm", - "5T2sXw0YokQqySkRaE0rJNOCWIdsSSlYOy39cIAWpV2+NEoAwkLotX5CqDdYPyWE6njjLQRdKVEWTTWq", - "ymHb1Mc6ta0Ecx4n/JvHfO0kN+J870SmEtxexrnxdp2XevgGJhXF0tynI/rcBzyPTdx92cWGzM/AWG5d", - "apJW5iB+CgQsdU0iU8umqwShm+ROFRjFJ9l7RFwbnT9PPvTa7GO6T/QVF6AKr0wUHB+y00cJlWzqBXvI", - "qG61zF/SCz/8D635XcQW/e33I0x6kGKedUpvA1rpfyzOoUvoLpyVbicI18oshCLMous5F4B07MPyGs+I", - "I/uJoeSjZhfSdoPUaIh5mlJpdlBlyh6xTR8Yen+KKlR8Jh7S7FwBJLMAvT9/bb9YFqPbsOdqlLbovymJ", - "6ZaA/rriZ8RmG52qZS59kICrJtncRx9fKT7p+wBjlYKUR/LMeM3dHKXetKZKFeBIuOKtYCWeChGJiDBA", - "2PJgUnLhUGw0eX+KPhxfvKmo7KmTuvrAK69+zea1JxSw5nrunkrA6hPdw26ieywRZFoBL349/vbFy9z3", - "5tyi+rS7Z89+wffdBaed9Dl+p8aQ5X5TnzxHnULXxFa9fC3Fjidv+78pF6llq127yelzahWvNLGvFez4", - "7BSnDlI8wl7dEtFOD1tkInaLFSBd4KR6noGQK/qZDcOYxNX57jOtiXWoNcH10liLyrwR5XrlUHweWkBo", - "m0sUrmFiw8wMlTNlHOIy9tZyJWtIywYBHZu1b7PdWlFsNrswNUZ1l74pheMHSZaVaHOnr6TZJfEq9rYJ", - "PgW6pCK/PKlAd/XPPmi6Jo7OfcjkeWdwBULpIM303W2Fn5+WoXEspXIRNa+O4ZrZxulJPW7x6nL13wAA", - "AP//pU/ObfRCAAA=", + "H4sIAAAAAAAC/+xbW3PbtrP/Khievh1actokM/V5qev04jaZ+NjJ5CFxNRCwklCBAAKAtvX36Lv/BxdS", + "vICypTit3elLGwvAYvHbxd6wvM2ILJQUIKzJjm4zQxZQYP/PY6X46lyWlon5OXwuwVj3s9JSgbYM/CRs", + "DJuLolrOLBT+H99omGVH2f+MN+THkfb4vBSWFeBIw3G9PlvnmV0pyI4yrDVeZet1nmn4XDINNDv62Nrq", + "sp4rp38C8YtPNGALF0RLzof51ZbNMPEjFAzRTFkmRXaUvT05RdUo0jADDYIAkhpxSTBHxhNGCttFlmdw", + "gwvFA7NhjRlRXTI6ms/HFoz1/zly/8lqXo3VTMwdr4z2GXgFSgPBFijCnGGDZlIjgQsYobd+jmPC4ikH", + "pAOCiNFxmHA6Q7Jg1gLNkV0AohgKKdAcBGhswSAsEKOjFuN/yqlJ8eYoJuB5GBb+L4wxozhe+dMhYxnn", + "iMgCDJppWUSkRytc8PtzbBQmCbZ/L6egBbj961ke2Ip/DUaWmoAZodO5kBoomq6QkOKgsXSKyRIENaPU", + "7vJagJ6kJBoVHfkZiFFUGqB+d1IaKwvQBzNMmJgj7e4CwqVdSM3+g9365F4a5sxYvZoQDRSEZZjvcO/i", + "4pN67d13rrouqQv3CjhYoOHG9a9aQKR3BGOxLf2EjWBpoNQ/cYcd5qZEAimOfhKm1LuYgB531eCEsnlc", + "PXB5B+/Nv+r5ONTzV8DcLs7BKCkM9PWgkDQhkZNSaxAWLfxqFJQN+blNUySXqfMrLecajOmTPYsjSIEm", + "ICyeBzlzialD2DHmcXUGbiZ1gW12lM24xM5/FPiGFWWRHT07PMyzgonw12HNgiiLKeh4vbSdUGwTZ/uw", + "ANG0zX6uv3b1jm7hgdOKLM9Eybmz9dmR1SXcdTc9RCk5vJZkeVFf+rYM4IbZCYmCGNiPCQvzcDiOjZ0E", + "kUzIAou5X1czz4R9+TxLLWwYHeGQ+5jpUgh3jDyjUnjZai11lmfXmLmIp3GUgQNHmkmuUjicSZ2wRi0J", + "7WJVVCRX68bLFy++e9HQjmcpIJSWVhLJm1AsrFVZ7v/n3Stxf5VU3Q2BZy6y0qCdPL2WBKizzh6oN1h5", + "W0wpC3HFWdtGD/y+zX409GydYKDPUTnlzCzen55hssRzGHQYPuTbEhB5d3OgpbQHGji27ArQ6BqbwgeL", + "I/QKZrjk1iArkdLsClsYU2bsGCsV5kmNlOOGtH8fJR1i7yAJw9k7w0IOODOFjbmWOu3SSgN6QAE7muDp", + "NxY0CKe0Ibqe42i/31bWbz+nPeR2aAA+O5phbiB/ODfktvTGs8HOVEoOWOzmoyIOzjQMmcipLAVN7ZN7", + "0CdMJTHxY5WN6NuBJYA65uwK3mk8mzGSpOENGyaWXTG7mmDbMrZNT7G71UpapmAg0usadqsv/5vJdGUD", + "XPdxBj6iSlKyPTQacMfBnfaq1sjldprXTFB5neZpl9MNGOga276xzqOGbQ5fI7RFY7vJe8KzW2cK+Db9", + "3N3hTYZHtylIMK5brkOpedrGbTs/E/N3WM8hcfr75QK73I67Dr/v3THAgVipt3ndvkp2UTGgrxiByf2c", + "xYBWTgbCiQ75LVo5lIpu9R7E143oTvZtIAX0BjMEkqnhZio2LMM7PVQilAoeCfQVUK/l98+6fFTql2P6", + "VvBVJ/jeODwpB5xvuAkPXv3LsxBYDWt9P6iPoszyRnhvrFTK/1ZF+FW14TIh2JJNVAgH73uQOn70YWep", + "6I7KlCpx1PoacW9jkW9Sj4butvbeckdqfocD3T4qO59qi0ltntZNyrNYU92R/70ThR4QKYv23jOzdzX5", + "pxvFGWGWrxAmBJQFilhRlKF4ar3bGKE3zBif/Wu0GZpvqs8GYQ1Ig2MKKJrCTGpAXqPdMpfKx0LRKMDo", + "fG52lP3x8Y8fPn0yl//7g1ngb1+8PPqID2aHB99f3r58vv4mJS24UX6PCRPGYs6BNipgA4mP0kCkCPgj", + "sgCyBIrwHDsSnjdMbIk5qkmiipDUqBQUtJ9VYOfnBBYEEJdk2TnKfU/wiGpP3iGSUjO7unAbxEAesAZ9", + "XIY7Fv76ubpKv3145y1TE+jfPrxDVi5BhLI485zZFVJaXjEK2psFR96FlJ7cBhef07sjuPXVnm3yFwup", + "7YHLASj6XIJeVZtJjT7A9EKSJVhEpBBAqsoUcwv95KyK1MIWm52xYr+Dg8u5STGTbmMihQ3XZN095Ctd", + "MopOXp8ijktBFv6hgKICC3fFkV/JBOgDX+Sk1TMMVu5+hYpZjjhbwicx968JzglqkyOKLZ5iAyb3BK9h", + "Wo2NPnl2mfWVvJqBLM/caGDrcPRsdOh9tgKBFcuOsu/8T8FaeYGOsWLjq2fjUCp0v8RYsH3CX8BWBbdW", + "UTHzxEPee0rDxFCz9PLyHt2XLv1m3x4eVkjGeLsBwfhPE8pHQZ/v0vZOZdSLqs1zmOG98IvD7/7CjS9C", + "pIdKga8wC+VAf5/KosB6FeHs4mjx3PgqRJBEngW8s0u3tBJT0BzTkFMb/tfM2Is45wvB3yUQijFr39z0", + "sKnq9dVB2rg49utnA1Ofo4ImjjSxcZG2SQDRfEjNgtEDY3+UdPVgipB6q123LawLQ9c9OTx7MBY68N8F", + "N6piyzbq4SAd3LfD3lfJMfh3Kx9fJCXSfNf6ShJJPZ3dSyKHf5tEAmpdiYSDdCSC4IYZG1yLjPELX4UH", + "ELOzuG4ZXQc77xKJvrjCw2gtLoU1LsCCdlvcBh8aQ97oQn3Q3wY6b4DWjaEvv6IQ2o+6dwuhSqbWefb8", + "8PnwI2OcLqRFM19vakstbLvTPcrTZvwXsE8T+R3V/0sRd370C82WuwdjF5eVath2/ejHv7JIHt4e3vVC", + "8dhsY4AZuaXxQrat4g2QsnHBotT2kjiRRYEFNePb+K/1sPTPSxF4PglTv4YG5EkipN5wJ0qdF3PMrE+I", + "fKodRA8URdrIyhrwKjkPGkCZmI8G8iWzEiRrctF9puo9KP2tVicUQmjXVnyh9TkvRddFbwS2l06KGZsP", + "xva1UzgJ8x6ha+jWEHqCOMPabBLgeOC+TVepaftiaiQHcy9Uw8xHiGu6Qtgqcg9jXh0MLWEVWq82jxZ9", + "6GMngCFSeSNRg7IH+HUVfzvyZzIEsI8O9l1S4Mbz985pMHJAVZWABw+JWtT3kuPnEkq4W47/76c9sYg1", + "9RDVl5c/mscQtuD9uTFrA/TnCEsF+DDOGjhgsxXp06oOfR7nPkK4O81hw02oGpQ0zEq9uvuVpTE3r2he", + "3sP61YAhgoUUjGCONrR8Ms2xsciUhIAxs9INe2hR3OVLowTA1IdemyeEeoPNU4IGIjUN5tlN7vKkylZu", + "U9XGtumSsXJbPeY8TPg3qfnaGW/A+d5ZTSW4vSx14yE7LXX/QUyskMW5T0f0qa95Hpu4h1KNlszPQBtm", + "bOyYlvogfBcENLZQIl3Lpq8EvrXkThUYh/fZe4RfrTagJx+HtZua7hOKhQWowisREodX7fiFQiWbesEe", + "Mqr7LtOX9MIN/0MLgBehX3/7/fCTHqSyZ6xU24CW6h+Ls28ZugtnqbrZwrXUSy4xNeh6wTggFZqynMZT", + "bPF+YijZuNmStN0gNbpjnqZUmu1UiRpI6NkHit6fogoVF/f5nDtVDUksQO/PX5svlsX41u+5Hscthm9K", + "ZLojoL+uEhqw2Uan6p+LXydkVcds6guQrxSfDH2NsY5ByiN5c7xmdoFio1pTpQqw2F/xTrASToWwQJhr", + "wHR1MC0Ztyh0nbw/RR+OL95UVPbUSVV97ZVWv2Yn2xMKWFMNeE8lYHVZ72E/6z0WCBJ9gRe/Hn/74mXq", + "43NmUH3a3VNpt+D7/oLTXi4dPlqjyDC3qUuhg06ha2yqxr6OYoeTd/3fjPHYv9Ut5KT0OfaNV5o41Bd2", + "fHaaxXbSbJw5dYtEew1tgYnQOlaAsJ6T6q0GfK7oZjYMYxRX7yPQuCYUpTYEN0tDYSrxYJRqnEPhrWgJ", + "vocuUriGqfEzE1TOpLaIidBoy6SoIS0bBFTo3L5Ntm4FsZnkwtgl1V/6puSWHURZVqJNnb6SZp/Eq9Do", + "xtkMyIrw9PKoAv3VP7ug6RpbsnAhk+OdwhVwqbw040e4FX5uWoLGsRDSBtScOvprZhqnx/W4ydaX6/8G", + "AAD//y1wpUoBQwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/core/domain/runtime_scroll.go b/internal/core/domain/runtime_scroll.go index effc21f4..24331abb 100644 --- a/internal/core/domain/runtime_scroll.go +++ b/internal/core/domain/runtime_scroll.go @@ -20,9 +20,8 @@ const ( RuntimeScrollStatusDeleted RuntimeScrollStatus = "deleted" ) -// Empty state denotes a legacy record whose digest may describe the source -// artifact rather than the installed release. Pending is persisted before a -// worker can mutate the volume; only its successful result confirms the cache. +// Pending is persisted before a worker can mutate the volume; only its +// successful result confirms the installed release. Missing state is unconfirmed. type InstalledReleaseState string const ( diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index b3020f5c..3a24f334 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -159,7 +159,6 @@ const ( RuntimeWorkerModeCreate RuntimeWorkerMode = "create" RuntimeWorkerModeUpdate RuntimeWorkerMode = "update" RuntimeWorkerModeRestore RuntimeWorkerMode = "restore" - RuntimeWorkerModeInspect RuntimeWorkerMode = "inspect" ) type RuntimeWorkerAction struct { diff --git a/internal/runtime/docker/workers.go b/internal/runtime/docker/workers.go index a3b2069b..ad9c017e 100644 --- a/internal/runtime/docker/workers.go +++ b/internal/runtime/docker/workers.go @@ -125,7 +125,7 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke action.NewDataOwner = image.Config.User } workerUser = "0" - } else if action.Mode != ports.RuntimeWorkerModeInspect { + } else { if err := b.prepareWritableRoot(ctx, root); err != nil { return nil, err } @@ -136,7 +136,7 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke if err != nil { return nil, err } - rootMount, err := DockerMount(root, action.MountPath, action.Mode == ports.RuntimeWorkerModeInspect, "") + rootMount, err := DockerMount(root, action.MountPath, false, "") if err != nil { return nil, err } diff --git a/internal/runtime/kubernetes/resources.go b/internal/runtime/kubernetes/resources.go index 314254c6..f026be10 100644 --- a/internal/runtime/kubernetes/resources.go +++ b/internal/runtime/kubernetes/resources.go @@ -90,20 +90,11 @@ func workerPullJobSpec(namespace string, jobName string, pvc string, image strin if action.Mode == ports.RuntimeWorkerModeUpdate { command = append(command, "--new-data-owner", "1000:1000") } - if action.Mode == ports.RuntimeWorkerModeInspect { - // No credential shell or recursive ownership changes for read-only inspection. - command = append([]string{"druid"}, command[4:]...) - } job := helperJobSpec(namespace, jobName, pvc, image, command, imagePullSecret, map[string]string{ labelComponent: "worker-pull", labelRuntimeID: runtimeLabel(action.RuntimeID), }) container := &job.Spec.Template.Spec.Containers[0] - if action.Mode == ports.RuntimeWorkerModeInspect { - for i := range container.VolumeMounts { - container.VolumeMounts[i].ReadOnly = true - } - } runAsRoot := int64(0) runAsNonRoot := false container.SecurityContext = &corev1.SecurityContext{ diff --git a/internal/runtime/kubernetes/resources_test.go b/internal/runtime/kubernetes/resources_test.go index 750fac4b..288e96d3 100644 --- a/internal/runtime/kubernetes/resources_test.go +++ b/internal/runtime/kubernetes/resources_test.go @@ -489,21 +489,6 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { } } -func TestInstalledReleaseInspectionMountsReadOnlyWithoutChown(t *testing.T) { - action := ports.RuntimeWorkerAction{Mode: ports.RuntimeWorkerModeInspect, RuntimeID: "inspect", Artifact: "registry.local/source:tag", MountPath: "/scroll"} - job := workerPullJobSpec("fixture", "inspect", "runtime-pvc", "druid:test", action, "", "", true, "druid-cli") - container := job.Spec.Template.Spec.Containers[0] - command := strings.Join(container.Command, " ") - if !strings.HasPrefix(command, "druid worker pull ") || strings.Contains(command, "chown") || !strings.Contains(command, "--mode inspect") { - t.Fatalf("unsafe inspection command: %s", command) - } - for _, mount := range container.VolumeMounts { - if !mount.ReadOnly { - t.Fatalf("inspection mount is writable: %#v", mount) - } - } -} - func TestSpawnPullWorkerCreateUsesFinalPVCAndWorkerJob(t *testing.T) { client := fake.NewSimpleClientset() backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, client)