From bc1bcc285dd5654b72293e1aaf41de081e21420f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Schottst=C3=A4dt?= Date: Sat, 22 Aug 2026 22:36:28 +0200 Subject: [PATCH 1/3] refactor(runtime): unify port services Keep one stable Kubernetes Service per logical port and select all exposing workloads through port labels. Preserve port Services across command and runtime stops. --- .../services/runtime_session_execution.go | 15 +- .../runtime_session_execution_test.go | 12 +- .../core/services/runtime_session_runtime.go | 4 +- .../core/services/runtime_supervisor_test.go | 4 +- internal/core/ports/services_ports.go | 17 +- internal/runtime/backend_factory_test.go | 4 +- internal/runtime/docker/backend_names_test.go | 24 +-- internal/runtime/docker/ports.go | 10 +- internal/runtime/docker/procedures.go | 4 +- internal/runtime/kubernetes/lifecycle.go | 15 +- internal/runtime/kubernetes/names.go | 4 + internal/runtime/kubernetes/ports.go | 116 ++++------- internal/runtime/kubernetes/procedures.go | 16 +- internal/runtime/kubernetes/resources.go | 33 ++-- internal/runtime/kubernetes/resources_test.go | 184 ++++++++++++------ test/mock/services.go | 16 +- 16 files changed, 253 insertions(+), 225 deletions(-) diff --git a/apps/druid/core/services/runtime_session_execution.go b/apps/druid/core/services/runtime_session_execution.go index 3f526127..81aa5886 100644 --- a/apps/druid/core/services/runtime_session_execution.go +++ b/apps/druid/core/services/runtime_session_execution.go @@ -70,14 +70,13 @@ func (s *RuntimeSession) runCommand(cmd string, extraEnv ...map[string]map[strin } exitCode, err := s.runtimeBackend.RunCommand(ports.RuntimeCommand{ - Name: cmd, - ScrollID: scrollID, - Command: command, - Root: root, - GlobalPorts: runtimePorts, - ReservedPorts: reservations, - Routing: routing, - ProcedureEnv: procedureEnv, + Name: cmd, + ScrollID: scrollID, + Command: command, + Root: root, + Ports: runtimePorts, + Routing: routing, + ProcedureEnv: procedureEnv, ProcedureStatusObserver: func(procedure string, status domain.ScrollLockStatus, exitCode *int) { s.persistProcedureStatus(cmd, procedure, status, exitCode) }, diff --git a/apps/druid/core/services/runtime_session_execution_test.go b/apps/druid/core/services/runtime_session_execution_test.go index 29fb92f1..fe327e3a 100644 --- a/apps/druid/core/services/runtime_session_execution_test.go +++ b/apps/druid/core/services/runtime_session_execution_test.go @@ -31,8 +31,8 @@ func TestRuntimeSessionRunCommandPassesCommandContextToRuntimeBackend(t *testing if seen.Root != session.runtimeScroll.Root { t.Fatalf("Root = %s, want %s", seen.Root, session.runtimeScroll.Root) } - if len(seen.GlobalPorts) != 1 || seen.GlobalPorts[0].Name != "http" { - t.Fatalf("GlobalPorts = %#v", seen.GlobalPorts) + if len(seen.Ports) != 1 || seen.Ports[0].Name != "http" { + t.Fatalf("Ports = %#v", seen.Ports) } } @@ -82,8 +82,8 @@ func TestRuntimeSessionRunCommandPassesRoutingAndScrollIdentity(t *testing.T) { if len(seen.Routing) != 1 || seen.Routing[0].PublicPort != 443 { t.Fatalf("Routing = %#v", seen.Routing) } - if len(seen.GlobalPorts) != 1 || seen.GlobalPorts[0].Port != 8080 { - t.Fatalf("GlobalPorts = %#v, fixed port must remain 8080", seen.GlobalPorts) + if len(seen.Ports) != 1 || seen.Ports[0].Port != 8080 { + t.Fatalf("Ports = %#v, fixed port must remain 8080", seen.Ports) } env := seen.ProcedureEnv["web"] if env["DRUID_SCROLL_ID"] != "scroll-a" || env["DRUID_SCROLL_NAME"] != "scroll-name" { @@ -123,8 +123,8 @@ func TestRuntimeSessionRunCommandResolvesDynamicPortsFromRouting(t *testing.T) { t.Fatal(err) } - if len(seen.GlobalPorts) != 1 || seen.GlobalPorts[0].Port != 11000 { - t.Fatalf("GlobalPorts = %#v, want dynamic port 11000", seen.GlobalPorts) + if len(seen.Ports) != 1 || seen.Ports[0].Port != 11000 { + t.Fatalf("Ports = %#v, want dynamic port 11000", seen.Ports) } env := seen.ProcedureEnv["server"] if env["DRUID_PORT_MAIN"] != "11000" || env["DRUID_PORT_MAIN_1"] != "11000" { diff --git a/apps/druid/core/services/runtime_session_runtime.go b/apps/druid/core/services/runtime_session_runtime.go index 6a539319..19747241 100644 --- a/apps/druid/core/services/runtime_session_runtime.go +++ b/apps/druid/core/services/runtime_session_runtime.go @@ -24,7 +24,7 @@ func (s *RuntimeSession) Ports() ([]domain.RuntimePortStatus, error) { if err != nil { return nil, err } - return s.runtimeBackend.ExpectedPorts(runtimeScroll.Root, file.Commands, runtimePorts, reservations) + return s.runtimeBackend.ExpectedPorts(runtimeScroll.Root, file.Commands, runtimePorts) } func (s *RuntimeSession) RoutingTargets() ([]domain.RuntimeRoutingTarget, error) { @@ -37,7 +37,7 @@ func (s *RuntimeSession) RoutingTargets() ([]domain.RuntimeRoutingTarget, error) if err != nil { return nil, err } - return s.runtimeBackend.RoutingTargets(runtimeScroll.Root, file.Commands, ports, reservations) + return s.runtimeBackend.RoutingTargets(runtimeScroll.Root, file.Commands, ports) } func (s *RuntimeSession) Queue() domain.ProcedureStatusMap { diff --git a/apps/druid/core/services/runtime_supervisor_test.go b/apps/druid/core/services/runtime_supervisor_test.go index 1c7325a7..71cf2b4d 100644 --- a/apps/druid/core/services/runtime_supervisor_test.go +++ b/apps/druid/core/services/runtime_supervisor_test.go @@ -1263,11 +1263,11 @@ func (f *fakeWorkerBackend) CreateUIPackageUpload(ctx context.Context, action po return ports.RuntimeUIPackageUpload{UploadURL: "http://uploads/" + action.RuntimeID + "/" + string(action.Scope), URL: "http://packages/" + action.RuntimeID + "/" + string(action.Scope) + "/app.wasm"}, nil } -func (f *fakeWorkerBackend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimePortStatus, error) { +func (f *fakeWorkerBackend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { return nil, nil } -func (f *fakeWorkerBackend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) { +func (f *fakeWorkerBackend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { return nil, nil } diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index 18d7f771..27a3fbcc 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -40,8 +40,8 @@ type RuntimeBackendInterface interface { RootRef(id string, namespace string) string RunCommand(command RuntimeCommand) (*int, error) CreateUIPackageUpload(ctx context.Context, action RuntimeUIPackageUploadAction) (RuntimeUIPackageUpload, error) - ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimePortStatus, error) - RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) + ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) + RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) StopRuntime(root string) error DeleteRuntime(root string, purgeData bool) error BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error @@ -92,12 +92,13 @@ type RuntimeScrollStore interface { } type RuntimeCommand struct { - Name string - ScrollID string - Command *domain.CommandInstructionSet - Root string - GlobalPorts []domain.Port - ReservedPorts []domain.Port + Name string + ScrollID string + Command *domain.CommandInstructionSet + Root string + // Ports is the complete runtime port set after reserved ports are merged + // and dynamic ports are resolved from routing assignments. + Ports []domain.Port Routing []domain.RuntimeRouteAssignment ProcedureEnv map[string]map[string]string ProcedureStatusObserver func(procedure string, status domain.ScrollLockStatus, exitCode *int) diff --git a/internal/runtime/backend_factory_test.go b/internal/runtime/backend_factory_test.go index ca069eec..36d9e0c0 100644 --- a/internal/runtime/backend_factory_test.go +++ b/internal/runtime/backend_factory_test.go @@ -101,11 +101,11 @@ func (f fakeBackend) CreateUIPackageUpload(ctx context.Context, action ports.Run return ports.RuntimeUIPackageUpload{}, nil } -func (f fakeBackend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimePortStatus, error) { +func (f fakeBackend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { return nil, nil } -func (f fakeBackend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) { +func (f fakeBackend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { return nil, nil } diff --git a/internal/runtime/docker/backend_names_test.go b/internal/runtime/docker/backend_names_test.go index 586c3862..086957e5 100644 --- a/internal/runtime/docker/backend_names_test.go +++ b/internal/runtime/docker/backend_names_test.go @@ -118,7 +118,7 @@ func TestRoutingTargetsUseFirstConcreteProcedureForSharedDockerPort(t *testing.T {Id: &coldstart, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, {Id: &start, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, }}, - }, []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}}, nil) + }, []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } @@ -135,27 +135,15 @@ func TestRoutingTargetsUseFirstConcreteProcedureForSharedDockerPort(t *testing.T t.Fatalf("main target missing: %#v", targets) } -func TestDockerRoutingTargetsIncludeUnclaimedRuntimePort(t *testing.T) { - reservedPorts := []domain.Port{{Name: "ssh", Port: 2222, Protocol: "tcp"}} - globalPorts := append(reservedPorts, domain.Port{Name: "unused-game-port", Port: 25565, Protocol: "tcp"}) - targets, err := (&Backend{}).RoutingTargets("docker-volume://druid-tools-data", nil, globalPorts, reservedPorts) +func TestDockerRoutingTargetsIncludeAllUnclaimedRuntimePorts(t *testing.T) { + ports := []domain.Port{{Name: "ssh", Port: 2222, Protocol: "tcp"}, {Name: "unused-game-port", Port: 25565, Protocol: "tcp"}} + targets, err := (&Backend{}).RoutingTargets("docker-volume://druid-tools-data", nil, ports) if err != nil { t.Fatal(err) } - for _, target := range targets { - if target.Name == "unused-game-port" { - t.Fatalf("ordinary unclaimed port was routed: %#v", target) - } - if target.Name == "ssh" && target.Procedure == "" && target.ServiceName == "" { - continue - } - } - for _, target := range targets { - if target.Name == "ssh" { - return - } + if len(targets) != 2 || targets[0].Name != "ssh" || targets[1].Name != "unused-game-port" { + t.Fatalf("targets = %#v, want every runtime port", targets) } - t.Fatalf("unclaimed ssh target missing: %#v", targets) } func TestContainerSpecAddsHostGatewayExtraHost(t *testing.T) { diff --git a/internal/runtime/docker/ports.go b/internal/runtime/docker/ports.go index 63413922..42088ee1 100644 --- a/internal/runtime/docker/ports.go +++ b/internal/runtime/docker/ports.go @@ -16,9 +16,9 @@ import ( "github.com/highcard-dev/daemon/internal/core/domain" ) -func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, _ []domain.Port) ([]domain.RuntimePortStatus, error) { +func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { statuses := []domain.RuntimePortStatus{} - portsByName := portsByName(globalPorts) + portsByName := portsByName(ports) for commandName, command := range commands { if command == nil { continue @@ -47,8 +47,8 @@ func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.Command return statuses, nil } -func (b *Backend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) { - portsByName := portsByName(globalPorts) +func (b *Backend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { + portsByName := portsByName(ports) targets := []domain.RuntimeRoutingTarget{} seen := map[string]struct{}{} commandNames := make([]string, 0, len(commands)) @@ -84,7 +84,7 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman } } } - for _, port := range reservedPorts { + for _, port := range ports { if _, ok := seen[port.Name]; ok { continue } diff --git a/internal/runtime/docker/procedures.go b/internal/runtime/docker/procedures.go index 15b7a69c..b964e840 100644 --- a/internal/runtime/docker/procedures.go +++ b/internal/runtime/docker/procedures.go @@ -46,14 +46,14 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { return nil, fmt.Errorf("docker runtime procedure %s requires image", procedureName) } command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) - if err := b.startPersistentContainer(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.GlobalPorts, command.Routing, env); err != nil { + if err := b.startPersistentContainer(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env); err != nil { command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) return nil, err } continue } command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) - exitCode, err := b.runProcedure(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.GlobalPorts, command.Routing, env) + exitCode, err := b.runProcedure(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env) if err != nil { if exitCode != nil && *exitCode != 0 && procedure.IgnoreFailure { command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) diff --git a/internal/runtime/kubernetes/lifecycle.go b/internal/runtime/kubernetes/lifecycle.go index b72612f7..e79b6d5c 100644 --- a/internal/runtime/kubernetes/lifecycle.go +++ b/internal/runtime/kubernetes/lifecycle.go @@ -61,15 +61,6 @@ func (b *Backend) StopCommand(root string, command string) error { if err := b.client.AppsV1().StatefulSets(namespace).DeleteCollection(ctx, options, listOptions); err != nil { return err } - services, err := b.client.CoreV1().Services(namespace).List(ctx, listOptions) - if err != nil { - return err - } - for _, service := range services.Items { - if err := b.client.CoreV1().Services(namespace).Delete(ctx, service.Name, options); err != nil && !apierrors.IsNotFound(err) { - return err - } - } if err := b.client.CoreV1().Pods(namespace).DeleteCollection(ctx, options, listOptions); err != nil { return err } @@ -82,7 +73,6 @@ func (b *Backend) StopCommand(root string, command string) error { jobs, jobErr := b.client.BatchV1().Jobs(namespace).List(waitCtx, listOptions) statefulSets, statefulSetErr := b.client.AppsV1().StatefulSets(namespace).List(waitCtx, listOptions) pods, podErr := b.client.CoreV1().Pods(namespace).List(waitCtx, listOptions) - services, serviceErr := b.client.CoreV1().Services(namespace).List(waitCtx, listOptions) if jobErr != nil { return jobErr } @@ -92,10 +82,7 @@ func (b *Backend) StopCommand(root string, command string) error { if podErr != nil { return podErr } - if serviceErr != nil { - return serviceErr - } - if len(jobs.Items) == 0 && len(statefulSets.Items) == 0 && len(pods.Items) == 0 && len(services.Items) == 0 { + if len(jobs.Items) == 0 && len(statefulSets.Items) == 0 && len(pods.Items) == 0 { return nil } select { diff --git a/internal/runtime/kubernetes/names.go b/internal/runtime/kubernetes/names.go index c847618a..457c94f0 100644 --- a/internal/runtime/kubernetes/names.go +++ b/internal/runtime/kubernetes/names.go @@ -50,6 +50,10 @@ func shortHash(value string) string { return hex.EncodeToString(sum[:])[:10] } +func portSelectorLabel(portName string) string { + return "druid.gg/port-" + shortHash(portName) +} + func dataPVCName(id string) string { return dnsLabel("druid-" + id + "-data") } diff --git a/internal/runtime/kubernetes/ports.go b/internal/runtime/kubernetes/ports.go index 74a0801d..63db33cc 100644 --- a/internal/runtime/kubernetes/ports.go +++ b/internal/runtime/kubernetes/ports.go @@ -17,20 +17,18 @@ import ( "go.uber.org/zap" ) -func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimePortStatus, error) { +func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { namespace, pvc, err := parseRef(root) if err != nil { return nil, err } - portsByName := portsByName(globalPorts) - reservedPortNames := portNames(reservedPorts) + portsByName := portsByName(ports) statuses := []domain.RuntimePortStatus{} now := time.Now() for commandName, command := range commands { if command == nil { continue } - portUse := expectedPortUse(command) for idx, procedure := range command.Procedures { if procedure == nil || len(procedure.ExpectedPorts) == 0 { continue @@ -39,7 +37,7 @@ func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.Command if procedure.Id != nil { procedureName = *procedure.Id } - traffic, trafficErr := b.procedureTrafficForSelector(context.Background(), namespace, serviceSelector(pvc, commandName, procedureName, "", nil), now) + traffic, trafficErr := b.procedureTrafficForSelector(context.Background(), namespace, procedureSelector(pvc, commandName, procedureName), now) for _, expectedPort := range procedure.ExpectedPorts { port := portsByName[expectedPort.Name] status := domain.RuntimePortStatus{ @@ -50,8 +48,7 @@ func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.Command KeepAliveTraffic: expectedPort.KeepAliveTraffic, Source: "kubernetes-service", } - serviceProcedure := serviceProcedureName(commandName, procedureName, expectedPort.Name, portUse, reservedPortNames) - serviceReady, hostPort := b.serviceReady(context.Background(), namespace, serviceName(root, serviceProcedure, expectedPort.Name)) + serviceReady, hostPort := b.serviceReady(context.Background(), namespace, portServiceName(root, expectedPort.Name)) status.Bound = serviceReady status.HostPort = hostPort if trafficErr != nil { @@ -89,13 +86,12 @@ func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.Command return statuses, nil } -func (b *Backend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) { +func (b *Backend) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { namespace, pvc, err := parseRef(root) if err != nil { return nil, err } - portsByName := portsByName(globalPorts) - reservedPortNames := portNames(reservedPorts) + portsByName := portsByName(ports) targets := []domain.RuntimeRoutingTarget{} seen := map[string]struct{}{} commandNames := make([]string, 0, len(commands)) @@ -108,7 +104,6 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman if command == nil { continue } - portUse := expectedPortUse(command) for idx, procedure := range command.Procedures { if procedure == nil || len(procedure.ExpectedPorts) == 0 { continue @@ -120,15 +115,6 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman } port := portsByName[expectedPort.Name] seen[expectedPort.Name] = struct{}{} - serviceProcedure := serviceProcedureName(commandName, procedureName, expectedPort.Name, portUse, reservedPortNames) - svcName := serviceName(root, serviceProcedure, expectedPort.Name) - selector := serviceSelector(pvc, commandName, procedureName, expectedPort.Name, portUse) - if currentSelector, ok := b.currentServiceSelector(context.Background(), namespace, svcName); ok { - selector = currentSelector - if currentProcedure := currentSelector[labelProcedure]; currentProcedure != "" { - procedureName = currentProcedure - } - } targets = append(targets, domain.RuntimeRoutingTarget{ Name: expectedPort.Name, Procedure: procedureName, @@ -136,13 +122,13 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman Port: port.Port, Protocol: normalizeProtocol(port.Protocol), Namespace: namespace, - ServiceName: svcName, - Selector: selector, + ServiceName: portServiceName(root, expectedPort.Name), + Selector: portServiceSelector(pvc, expectedPort.Name), }) } } } - for _, port := range reservedPorts { + for _, port := range ports { if _, ok := seen[port.Name]; ok { continue } @@ -152,30 +138,24 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman Port: port.Port, Protocol: normalizeProtocol(port.Protocol), Namespace: namespace, - ServiceName: reservedServiceName(root, port.Name), - Selector: map[string]string{ - labelManagedBy: "druid", - labelComponent: "runtime", - labelScrollID: dnsLabel(pvc), - labelPortName: dnsLabel(port.Name), - }, + ServiceName: portServiceName(root, port.Name), + Selector: portServiceSelector(pvc, port.Name), }) } sort.Slice(targets, func(i, j int) bool { return targets[i].Name < targets[j].Name }) return targets, nil } -func (b *Backend) ensureExpectedServices(ctx context.Context, root string, commandName string, procedureName string, procedure *domain.Procedure, globalPorts []domain.Port, portUse map[string]int, reservedPortNames map[string]struct{}) error { +func (b *Backend) ensureExpectedServices(ctx context.Context, root string, commandName string, procedureName string, procedure *domain.Procedure, ports []domain.Port) error { namespace, _, err := parseRef(root) if err != nil { logger.Log().Error("Cannot reconcile Kubernetes Services for invalid root", zap.String("root", root), zap.String("command", commandName), zap.String("procedure", procedureName), zap.Error(err)) return err } - ports := portsByName(globalPorts) + portsByName := portsByName(ports) for _, expected := range procedure.ExpectedPorts { - port := ports[expected.Name] - serviceProcedure := serviceProcedureName(commandName, procedureName, expected.Name, portUse, reservedPortNames) - service, err := serviceSpec(namespace, root, serviceProcedure, serviceSelector(refPVCName(root), commandName, procedureName, expected.Name, portUse), expected.Name, port) + port := portsByName[expected.Name] + service, err := serviceSpec(namespace, root, portServiceSelector(refPVCName(root), expected.Name), expected.Name, port) if err != nil { logger.Log().Error("Failed to build Kubernetes Service for expected port", zap.String("namespace", namespace), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("port", expected.Name), zap.Error(err)) return err @@ -184,7 +164,6 @@ func (b *Backend) ensureExpectedServices(ctx context.Context, root string, comma zap.String("namespace", namespace), zap.String("command", commandName), zap.String("procedure", procedureName), - zap.String("service_procedure", serviceProcedure), zap.String("service", service.Name), zap.String("port_name", expected.Name), zap.Int("port", port.Port), @@ -214,57 +193,52 @@ func (b *Backend) ensureExpectedServices(ctx context.Context, root string, comma return err } } + if err := b.deleteLegacyPortServices(ctx, namespace, root, expected.Name, service.Name); err != nil { + return err + } } return nil } -func expectedPortUse(command *domain.CommandInstructionSet) map[string]int { - use := map[string]int{} - if command == nil { - return use +func (b *Backend) deleteLegacyPortServices(ctx context.Context, namespace string, root string, portName string, stableName string) error { + _, pvc, err := parseRef(root) + if err != nil { + return err } - for _, procedure := range command.Procedures { - if procedure == nil { + selector := labels.SelectorFromSet(labels.Set{ + labelScrollID: dnsLabel(pvc), + labelPortName: dnsLabel(portName), + }).String() + services, err := b.client.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return err + } + for _, service := range services.Items { + if service.Name == stableName { continue } - for _, expected := range procedure.ExpectedPorts { - use[expected.Name]++ + if err := b.client.CoreV1().Services(namespace).Delete(ctx, service.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err } } - return use -} - -func serviceProcedureName(commandName string, procedureName string, portName string, portUse map[string]int, reservedPortNames map[string]struct{}) string { - if _, reserved := reservedPortNames[portName]; reserved { - return portName - } - if portUse[portName] > 1 { - return commandName - } - return procedureName + return nil } -func reservedServiceName(root string, portName string) string { +func portServiceName(root string, portName string) string { return serviceName(root, portName, portName) } -func serviceSelector(pvc string, commandName string, procedureName string, portName string, portUse map[string]int) map[string]string { +func procedureSelector(pvc string, commandName string, procedureName string) map[string]string { selector := baseLabels(pvc) selector[labelCommand] = dnsLabel(commandName) selector[labelProcedure] = dnsLabel(procedureName) return selector } -func (b *Backend) currentServiceSelector(ctx context.Context, namespace string, name string) (map[string]string, bool) { - service, err := b.client.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil || len(service.Spec.Selector) == 0 { - return nil, false - } - selector := make(map[string]string, len(service.Spec.Selector)) - for key, value := range service.Spec.Selector { - selector[key] = value - } - return selector, true +func portServiceSelector(pvc string, portName string) map[string]string { + selector := baseLabels(pvc) + selector[portSelectorLabel(portName)] = "true" + return selector } func (b *Backend) serviceReady(ctx context.Context, namespace string, name string) (bool, int) { @@ -303,14 +277,6 @@ func portsByName(ports []domain.Port) map[string]domain.Port { return result } -func portNames(ports []domain.Port) map[string]struct{} { - result := make(map[string]struct{}, len(ports)) - for _, port := range ports { - result[port.Name] = struct{}{} - } - return result -} - func normalizeProtocol(protocol string) string { switch strings.ToLower(protocol) { case "", "tcp": diff --git a/internal/runtime/kubernetes/procedures.go b/internal/runtime/kubernetes/procedures.go index e046bd8d..d1eb6ffc 100644 --- a/internal/runtime/kubernetes/procedures.go +++ b/internal/runtime/kubernetes/procedures.go @@ -27,8 +27,6 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { zap.String("root", command.Root), zap.Int("procedures", len(command.Command.Procedures)), ) - portUse := expectedPortUse(command.Command) - reservedPortNames := portNames(command.ReservedPorts) startIndex := 0 if command.Command.Run == domain.RunModeRestart { resumeIndex, err := b.resumeRestartProcedureIndex(context.Background(), command.Root, command.Name, command.Command) @@ -101,7 +99,7 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { return nil, err } command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) - if err := b.ensurePersistentProcedure(context.Background(), command.ScrollID, command.Root, command.Name, procedureName, resourceName, procedure, command.GlobalPorts, env, portUse, reservedPortNames); err != nil { + if err := b.ensurePersistentProcedure(context.Background(), command.ScrollID, command.Root, command.Name, procedureName, resourceName, procedure, command.Ports, env); err != nil { command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) logger.Log().Error("Kubernetes persistent procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Error(err)) return nil, err @@ -109,7 +107,7 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { continue } command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) - exitCode, err := b.runJobProcedure(command.ScrollID, command.Name, procedureName, resourceName, procedure, command.Root, command.GlobalPorts, env, portUse, reservedPortNames) + exitCode, err := b.runJobProcedure(command.ScrollID, command.Name, procedureName, resourceName, procedure, command.Root, command.Ports, env) if err != nil { if exitCode != nil && *exitCode != 0 && procedure.IgnoreFailure { command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) @@ -139,7 +137,7 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { return nil, nil } -func (b *Backend) runJobProcedure(scrollID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, env map[string]string, portUse map[string]int, reservedPortNames map[string]struct{}) (*int, error) { +func (b *Backend) runJobProcedure(scrollID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, ports []domain.Port, env map[string]string) (*int, error) { if procedure.IsSignal() { logger.Log().Info("Running Kubernetes signal procedure", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal)) if err := b.Signal(procedureName, procedure.Target, procedure.Signal, root); err != nil { @@ -154,7 +152,7 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure return nil, err } ctx := context.Background() - if err := b.ensureExpectedServices(ctx, root, commandName, procedureName, procedure, globalPorts, portUse, reservedPortNames); err != nil { + if err := b.ensureExpectedServices(ctx, root, commandName, procedureName, procedure, ports); err != nil { logger.Log().Error("Failed to reconcile Kubernetes procedure Services", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.Error(err)) return nil, err } @@ -201,7 +199,7 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure } else { logger.Log().Warn("Could not find Kubernetes job pod before wait; console logs may be empty", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.Error(err)) } - exitCode, err := b.waitForJobWithIdleStop(ctx, namespace, jobName, b.keepAliveTrafficIdleStopper(namespace, root, commandName, procedureName, procedure, globalPorts)) + exitCode, err := b.waitForJobWithIdleStop(ctx, namespace, jobName, b.keepAliveTrafficIdleStopper(namespace, root, commandName, procedureName, procedure, ports)) if exitCode != nil { console.MarkExited(*exitCode) } @@ -228,8 +226,8 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure return exitCode, nil } -func (b *Backend) ensurePersistentProcedure(ctx context.Context, scrollID string, root string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, globalPorts []domain.Port, env map[string]string, portUse map[string]int, reservedPortNames map[string]struct{}) error { - if err := b.ensureExpectedServices(ctx, root, commandName, procedureName, procedure, globalPorts, portUse, reservedPortNames); err != nil { +func (b *Backend) ensurePersistentProcedure(ctx context.Context, scrollID string, root string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, ports []domain.Port, env map[string]string) error { + if err := b.ensureExpectedServices(ctx, root, commandName, procedureName, procedure, ports); err != nil { logger.Log().Error("Failed to reconcile Kubernetes persistent procedure Services", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.Error(err)) return err } diff --git a/internal/runtime/kubernetes/resources.go b/internal/runtime/kubernetes/resources.go index 18eb9059..e9a0718c 100644 --- a/internal/runtime/kubernetes/resources.go +++ b/internal/runtime/kubernetes/resources.go @@ -188,6 +188,9 @@ func procedureJobSpec(namespace string, root string, commandName string, procedu labels[labelProcedure] = dnsLabel(procedureName) labels[labelCommand] = dnsLabel(commandName) labels[labelAttempt] = fmt.Sprintf("%d", attempt) + for _, expectedPort := range procedure.ExpectedPorts { + labels[portSelectorLabel(expectedPort.Name)] = "true" + } if len(procedure.ExpectedPorts) == 1 { labels[labelPortName] = dnsLabel(procedure.ExpectedPorts[0].Name) } @@ -232,10 +235,14 @@ func procedureStatefulSetSpec(namespace string, root string, commandName string, if err != nil { return nil, err } - labels := baseLabels(pvc) - labels[labelRuntimeID] = dnsLabel(runtimeID(root)) - labels[labelProcedure] = dnsLabel(procedureName) - labels[labelCommand] = dnsLabel(commandName) + selectorLabels := baseLabels(pvc) + selectorLabels[labelRuntimeID] = dnsLabel(runtimeID(root)) + selectorLabels[labelProcedure] = dnsLabel(procedureName) + selectorLabels[labelCommand] = dnsLabel(commandName) + labels := copyLabels(selectorLabels) + for _, expectedPort := range procedure.ExpectedPorts { + labels[portSelectorLabel(expectedPort.Name)] = "true" + } if len(procedure.ExpectedPorts) == 1 { labels[labelPortName] = dnsLabel(procedure.ExpectedPorts[0].Name) } @@ -267,7 +274,7 @@ func procedureStatefulSetSpec(namespace string, root string, commandName string, Spec: appsv1.StatefulSetSpec{ Replicas: &replicas, ServiceName: resourceName, - Selector: &metav1.LabelSelector{MatchLabels: labels}, + Selector: &metav1.LabelSelector{MatchLabels: selectorLabels}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: podSpec, @@ -276,24 +283,20 @@ func procedureStatefulSetSpec(namespace string, root string, commandName string, }, nil } -func serviceSpec(namespace string, root string, serviceProcedure string, selector map[string]string, portName string, port domain.Port) (*corev1.Service, error) { +func serviceSpec(namespace string, root string, selector map[string]string, portName string, port domain.Port) (*corev1.Service, error) { _, pvc, err := parseRef(root) if err != nil { return nil, err } labels := baseLabels(pvc) - labels[labelProcedure] = dnsLabel(serviceProcedure) labels[labelPortName] = dnsLabel(portName) - if command := selector[labelCommand]; command != "" { - labels[labelCommand] = command - } protocol := corev1.ProtocolTCP if normalizeProtocol(port.Protocol) == "udp" { protocol = corev1.ProtocolUDP } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceName(root, serviceProcedure, portName), + Name: portServiceName(root, portName), Namespace: namespace, Labels: labels, }, @@ -310,6 +313,14 @@ func serviceSpec(namespace string, root string, serviceProcedure string, selecto }, nil } +func copyLabels(source map[string]string) map[string]string { + result := make(map[string]string, len(source)) + for key, value := range source { + result[key] = value + } + return result +} + func pvcVolume(name string, pvc string) corev1.Volume { return corev1.Volume{ Name: name, diff --git a/internal/runtime/kubernetes/resources_test.go b/internal/runtime/kubernetes/resources_test.go index 27eca2c7..9b5cfa91 100644 --- a/internal/runtime/kubernetes/resources_test.go +++ b/internal/runtime/kubernetes/resources_test.go @@ -282,6 +282,37 @@ func TestProcedureStatefulSetSpecBuildsPersistentWorkload(t *testing.T) { } } +func TestProcedureWorkloadsLabelEveryExpectedPort(t *testing.T) { + procedure := &domain.Procedure{ + Image: "nginx:1.27", + ExpectedPorts: []domain.ExpectedPort{ + {Name: "http"}, + {Name: "admin"}, + }, + } + root := ref("druid", "druid-static-web-data") + job, err := procedureJobSpec("druid", root, "start", "web", "static-web-start-0", 1, procedure, nil, "", "") + if err != nil { + t.Fatal(err) + } + statefulSet, err := procedureStatefulSetSpec("druid", root, "start", "web", "static-web-start-0", procedure, nil, "") + if err != nil { + t.Fatal(err) + } + for _, portName := range []string{"http", "admin"} { + label := portSelectorLabel(portName) + if job.Spec.Template.Labels[label] != "true" { + t.Fatalf("job labels = %#v, want %s", job.Spec.Template.Labels, label) + } + if statefulSet.Spec.Template.Labels[label] != "true" { + t.Fatalf("StatefulSet pod labels = %#v, want %s", statefulSet.Spec.Template.Labels, label) + } + if _, immutable := statefulSet.Spec.Selector.MatchLabels[label]; immutable { + t.Fatalf("StatefulSet selector unexpectedly contains mutable port label %s", label) + } + } +} + func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { action := ports.RuntimeWorkerAction{ Mode: ports.RuntimeWorkerModeUpdate, @@ -707,14 +738,14 @@ func TestExpectedServicesUseRootNamespace(t *testing.T) { root := ref("games", dataPVCName("deployment-123")) procedure := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "http"}}} - err := backend.ensureExpectedServices(context.Background(), root, "start", "start", procedure, []domain.Port{{Name: "http", Port: 8080, Protocol: "tcp"}}, map[string]int{"http": 1}, nil) + err := backend.ensureExpectedServices(context.Background(), root, "start", "start", procedure, []domain.Port{{Name: "http", Port: 8080, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } - if _, err := client.CoreV1().Services("games").Get(context.Background(), serviceName(root, "start", "http"), metav1.GetOptions{}); err != nil { + if _, err := client.CoreV1().Services("games").Get(context.Background(), portServiceName(root, "http"), metav1.GetOptions{}); err != nil { t.Fatalf("service in runtime namespace: %v", err) } - if _, err := client.CoreV1().Services("druid-system").Get(context.Background(), serviceName(root, "start", "http"), metav1.GetOptions{}); !apierrors.IsNotFound(err) { + if _, err := client.CoreV1().Services("druid-system").Get(context.Background(), portServiceName(root, "http"), metav1.GetOptions{}); !apierrors.IsNotFound(err) { t.Fatalf("service in backend namespace error = %v, want not found", err) } } @@ -758,7 +789,7 @@ func TestExpectedPortsUsesPodStatsTraffic(t *testing.T) { globalPodTrafficStore.record("pod-start-stats", 100, 50, time.Now().Add(-6*time.Minute)) globalPodTrafficStore.record("pod-start-stats", 100, 50, time.Now().Add(-4*time.Minute)) setStatsReader(backend, "node-a", podStats("druid", "pod-start-stats", 200, 75), nil) - service, err := serviceSpec("druid", root, procedureName, serviceSelector(refPVCName(root), procedureName, procedureName, "http", map[string]int{"http": 1}), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) if err != nil { t.Fatal(err) } @@ -783,7 +814,7 @@ func TestExpectedPortsUsesPodStatsTraffic(t *testing.T) { Id: &procedureName, ExpectedPorts: []domain.ExpectedPort{{Name: "http", KeepAliveTraffic: "1b/5m"}}, }}}, - }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}, nil) + }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } @@ -811,7 +842,7 @@ func TestExpectedPortsDegradesWhenPodStatsUnavailable(t *testing.T) { t.Fatal(err) } setStatsReader(backend, "node-a", nil, errors.New("stats unavailable")) - service, err := serviceSpec("druid", root, "start", serviceSelector(refPVCName(root), "start", "start", "http", map[string]int{"http": 1}), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) if err != nil { t.Fatal(err) } @@ -833,7 +864,7 @@ func TestExpectedPortsDegradesWhenPodStatsUnavailable(t *testing.T) { statuses, err := backend.ExpectedPorts(root, map[string]*domain.CommandInstructionSet{ "start": {Procedures: []*domain.Procedure{{Id: ptrString("start"), ExpectedPorts: []domain.ExpectedPort{{Name: "http", KeepAliveTraffic: "1b/5m"}}}}}, - }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}, nil) + }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } @@ -860,7 +891,7 @@ func TestExpectedPortsWithoutActivePodDoesNotBorrowTraffic(t *testing.T) { } globalPodTrafficStore.record("pod-other", 100, 50, time.Now().Add(-6*time.Minute)) setStatsReader(backend, "node-a", podStats("druid", "pod-other", 200, 75), nil) - service, err := serviceSpec("druid", root, procedureName, serviceSelector(refPVCName(root), procedureName, procedureName, "http", map[string]int{"http": 1}), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 80, Protocol: "tcp"}) if err != nil { t.Fatal(err) } @@ -882,7 +913,7 @@ func TestExpectedPortsWithoutActivePodDoesNotBorrowTraffic(t *testing.T) { statuses, err := backend.ExpectedPorts(root, map[string]*domain.CommandInstructionSet{ "start": {Procedures: []*domain.Procedure{{Id: &procedureName, ExpectedPorts: []domain.ExpectedPort{{Name: "http", KeepAliveTraffic: "1b/5m"}}}}}, - }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}, nil) + }, []domain.Port{{Name: "http", Port: 80, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } @@ -1081,7 +1112,7 @@ func TestRoutingTargetsReturnStableBackendServices(t *testing.T) { Id: &procedureID, ExpectedPorts: []domain.ExpectedPort{{Name: "http"}}, }}}, - }, []domain.Port{{Name: "http", Port: 8080, Protocol: "http"}}, nil) + }, []domain.Port{{Name: "http", Port: 8080, Protocol: "http"}}) if err != nil { t.Fatal(err) } @@ -1095,65 +1126,59 @@ func TestRoutingTargetsReturnStableBackendServices(t *testing.T) { target = item } } - if target.Namespace != "druid" || target.ServiceName != serviceName(root, "web", "http") || target.Port != 8080 { + if target.Namespace != "druid" || target.ServiceName != portServiceName(root, "http") || target.Port != 8080 { t.Fatalf("target = %#v", target) } if target.Protocol != "http" || target.PortName != "http" || target.Procedure != "web" { t.Fatalf("target = %#v", target) } - if target.Selector[labelScrollID] != "druid-static-web-data" || target.Selector[labelProcedure] != "web" { + if target.Selector[labelScrollID] != "druid-static-web-data" || target.Selector[portSelectorLabel("http")] != "true" { t.Fatalf("selector = %#v", target.Selector) } } -func TestRoutingTargetsIncludeUnclaimedRuntimePort(t *testing.T) { +func TestRoutingTargetsIncludeAllUnclaimedRuntimePorts(t *testing.T) { backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) root := ref("druid", "druid-static-web-data") - reservedPorts := []domain.Port{{Name: "vscode", Port: 3333, Protocol: "http"}} - globalPorts := append(reservedPorts, domain.Port{Name: "unused-game-port", Port: 25565, Protocol: "tcp"}) - targets, err := backend.RoutingTargets(root, nil, globalPorts, reservedPorts) + ports := []domain.Port{{Name: "vscode", Port: 3333, Protocol: "http"}, {Name: "unused-game-port", Port: 25565, Protocol: "tcp"}} + targets, err := backend.RoutingTargets(root, nil, ports) if err != nil { t.Fatal(err) } + if len(targets) != 2 { + t.Fatalf("targets = %#v, want every runtime port", targets) + } for _, target := range targets { - if target.Name != "vscode" { - if target.Name == "unused-game-port" { - t.Fatalf("ordinary unclaimed port was routed: %#v", target) - } - continue - } - if target.Procedure != "" || target.ServiceName != reservedServiceName(root, "vscode") || target.Selector[labelPortName] != "vscode" { + if target.Procedure != "" || target.ServiceName != portServiceName(root, target.Name) || target.Selector[portSelectorLabel(target.Name)] != "true" { t.Fatalf("unclaimed target = %#v", target) } - return } - t.Fatalf("vscode target missing: %#v", targets) } -func TestReservedPortRoutingTargetsStableServiceCreatedByCommand(t *testing.T) { +func TestPortRoutingTargetUsesStableServiceCreatedByCommand(t *testing.T) { client := fake.NewSimpleClientset() backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) root := ref("druid", "druid-static-web-data") - reservedPorts := []domain.Port{{Name: "webdav", Port: 8084, Protocol: "http"}} + ports := []domain.Port{{Name: "webdav", Port: 8084, Protocol: "http"}} - before, err := backend.RoutingTargets(root, nil, reservedPorts, reservedPorts) + before, err := backend.RoutingTargets(root, nil, ports) if err != nil { t.Fatal(err) } - wantService := reservedServiceName(root, "webdav") + wantService := portServiceName(root, "webdav") if len(before) != 1 || before[0].ServiceName != wantService { t.Fatalf("unclaimed targets = %#v, want backend Service %q", before, wantService) } procedure := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "webdav"}}} - if err := backend.ensureExpectedServices(context.Background(), root, "druid_ui_dev", "files", procedure, reservedPorts, map[string]int{"webdav": 1}, portNames(reservedPorts)); err != nil { + if err := backend.ensureExpectedServices(context.Background(), root, "druid_ui_dev", "files", procedure, ports); err != nil { t.Fatal(err) } service, err := client.CoreV1().Services("druid").Get(context.Background(), wantService, metav1.GetOptions{}) if err != nil { t.Fatal(err) } - if service.Spec.Selector[labelCommand] != "druid-ui-dev" || service.Spec.Selector[labelProcedure] != "files" { + if service.Spec.Selector[portSelectorLabel("webdav")] != "true" || service.Spec.Selector[labelCommand] != "" || service.Spec.Selector[labelProcedure] != "" { t.Fatalf("service selector = %#v", service.Spec.Selector) } @@ -1162,7 +1187,7 @@ func TestReservedPortRoutingTargetsStableServiceCreatedByCommand(t *testing.T) { Id: ptrString("files"), ExpectedPorts: []domain.ExpectedPort{{Name: "webdav"}}, }}}, - }, reservedPorts, reservedPorts) + }, ports) if err != nil { t.Fatal(err) } @@ -1175,8 +1200,7 @@ func TestServiceSpecUsesRuntimePortForServiceAndTarget(t *testing.T) { service, err := serviceSpec( "druid", ref("druid", "druid-static-game-data"), - "start", - map[string]string{"druid.gg/procedure": "start"}, + portServiceSelector("druid-static-game-data", "main"), "main", domain.Port{Name: "main", Port: 7777, Protocol: "udp"}, ) @@ -1201,7 +1225,7 @@ func TestRoutingTargetsCollapseColdstarterAndRuntimePort(t *testing.T) { {Id: &coldstart, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, {Id: &start, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, }}, - }, []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}}, nil) + }, []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}}) if err != nil { t.Fatal(err) } @@ -1218,61 +1242,95 @@ func TestRoutingTargetsCollapseColdstarterAndRuntimePort(t *testing.T) { if mainTargets[0].Name != "main" || mainTargets[0].Procedure != "coldstart" { t.Fatalf("main target = %#v", mainTargets[0]) } - if mainTargets[0].Selector[labelCommand] != "start" || mainTargets[0].Selector[labelProcedure] != "coldstart" { + if mainTargets[0].Selector[portSelectorLabel("main")] != "true" || mainTargets[0].Selector[labelCommand] != "" || mainTargets[0].Selector[labelProcedure] != "" { t.Fatalf("selector = %#v", mainTargets[0].Selector) } } -func TestExpectedServiceForSharedPortMovesToActiveProcedure(t *testing.T) { +func TestExpectedServiceForSharedPortKeepsStableLabelSelector(t *testing.T) { client := fake.NewSimpleClientset() backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) root := ref("druid", "druid-minecraft-data") coldstart := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}} start := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}} ports := []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}} - portUse := map[string]int{"main": 2} - if err := backend.ensureExpectedServices(context.Background(), root, "start", "coldstart", coldstart, ports, portUse, nil); err != nil { + if err := backend.ensureExpectedServices(context.Background(), root, "start", "coldstart", coldstart, ports); err != nil { t.Fatal(err) } - service, err := client.CoreV1().Services("druid").Get(context.Background(), serviceName(root, "start", "main"), metav1.GetOptions{}) + service, err := client.CoreV1().Services("druid").Get(context.Background(), portServiceName(root, "main"), metav1.GetOptions{}) if err != nil { t.Fatal(err) } - if service.Spec.Selector[labelProcedure] != "coldstart" { + if service.Spec.Selector[portSelectorLabel("main")] != "true" || service.Spec.Selector[labelProcedure] != "" { t.Fatalf("coldstart selector = %#v", service.Spec.Selector) } - if err := backend.ensureExpectedServices(context.Background(), root, "start", "start", start, ports, portUse, nil); err != nil { + if err := backend.ensureExpectedServices(context.Background(), root, "start", "start", start, ports); err != nil { t.Fatal(err) } - service, err = client.CoreV1().Services("druid").Get(context.Background(), serviceName(root, "start", "main"), metav1.GetOptions{}) + service, err = client.CoreV1().Services("druid").Get(context.Background(), portServiceName(root, "main"), metav1.GetOptions{}) if err != nil { t.Fatal(err) } - if service.Spec.Selector[labelProcedure] != "start" { + if service.Spec.Selector[portSelectorLabel("main")] != "true" || service.Spec.Selector[labelProcedure] != "" { t.Fatalf("start selector = %#v", service.Spec.Selector) } } -func TestRoutingTargetsUseCurrentServiceSelector(t *testing.T) { - client := fake.NewSimpleClientset() +func TestStablePortServiceSelectsEveryCollidingProcedure(t *testing.T) { + root := ref("druid", "druid-minecraft-data") + port := domain.Port{Name: "main", Port: 25565, Protocol: "tcp"} + procedure := &domain.Procedure{Image: "busybox", ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}} + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "main"), "main", port) + if err != nil { + t.Fatal(err) + } + coldstart, err := procedureJobSpec("druid", root, "start", "coldstart", "minecraft-coldstart", 1, procedure, nil, "", "") + if err != nil { + t.Fatal(err) + } + runtime, err := procedureStatefulSetSpec("druid", root, "start", "runtime", "minecraft-runtime", procedure, nil, "") + if err != nil { + t.Fatal(err) + } + portLabel := portSelectorLabel("main") + if service.Spec.Selector[portLabel] != "true" || coldstart.Spec.Template.Labels[portLabel] != "true" || runtime.Spec.Template.Labels[portLabel] != "true" { + t.Fatalf("selector = %#v, coldstart labels = %#v, runtime labels = %#v", service.Spec.Selector, coldstart.Spec.Template.Labels, runtime.Spec.Template.Labels) + } +} + +func TestExpectedServiceReplacesLegacyProcedureNamedService(t *testing.T) { + root := ref("druid", "druid-static-web-data") + legacyName := serviceName(root, "web", "http") + legacyLabels := baseLabels(refPVCName(root)) + legacyLabels[labelPortName] = "http" + client := fake.NewSimpleClientset(&corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: legacyName, Namespace: "druid", Labels: legacyLabels}}) backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + procedure := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "http"}}} + if err := backend.ensureExpectedServices(context.Background(), root, "start", "web", procedure, []domain.Port{{Name: "http", Port: 8080, Protocol: "tcp"}}); err != nil { + t.Fatal(err) + } + if _, err := client.CoreV1().Services("druid").Get(context.Background(), legacyName, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("legacy Service error = %v, want not found", err) + } + if _, err := client.CoreV1().Services("druid").Get(context.Background(), portServiceName(root, "http"), metav1.GetOptions{}); err != nil { + t.Fatalf("stable Service error = %v", err) + } +} + +func TestRoutingTargetsUseStablePortSelector(t *testing.T) { + backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) root := ref("druid", "druid-minecraft-data") coldstart := "coldstart" start := "start" ports := []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}} - portUse := map[string]int{"main": 2} - if err := backend.ensureExpectedServices(context.Background(), root, "start", "start", &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, ports, portUse, nil); err != nil { - t.Fatal(err) - } - targets, err := backend.RoutingTargets(root, map[string]*domain.CommandInstructionSet{ "start": {Procedures: []*domain.Procedure{ {Id: &coldstart, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, {Id: &start, ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, }}, - }, ports, nil) + }, ports) if err != nil { t.Fatal(err) } @@ -1286,7 +1344,7 @@ func TestRoutingTargetsUseCurrentServiceSelector(t *testing.T) { if main == nil { t.Fatalf("targets = %#v", targets) } - if main.Procedure != "start" || main.Selector[labelProcedure] != "start" { + if main.Procedure != "coldstart" || main.Selector[portSelectorLabel("main")] != "true" || main.Selector[labelProcedure] != "" { t.Fatalf("main target = %#v", main) } } @@ -1299,7 +1357,7 @@ func TestStopRuntimeDeletesWorkloadsButPreservesDataAndServices(t *testing.T) { labels[labelProcedure] = "web" jobName := "static-web-web-0" statefulSetName := "static-web-web-0" - service, err := serviceSpec("druid", root, "web", serviceSelector(refPVCName(root), "web", "web", "http", map[string]int{"http": 1}), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) if err != nil { t.Fatal(err) } @@ -1350,11 +1408,27 @@ func TestStopRuntimeDeletesWorkloadsButPreservesDataAndServices(t *testing.T) { } } +func TestStopCommandPreservesStablePortService(t *testing.T) { + root := ref("druid", "druid-static-web-data") + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) + if err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset(service) + backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + if err := backend.StopCommand(root, "start"); err != nil { + t.Fatal(err) + } + if _, err := client.CoreV1().Services("druid").Get(context.Background(), service.Name, metav1.GetOptions{}); err != nil { + t.Fatalf("stable Service error = %v, want preserved", err) + } +} + func TestDeleteRuntimePurgesServicesAndDataWhenRequested(t *testing.T) { client := fake.NewSimpleClientset() backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) root := ref("druid", "druid-static-web-data") - service, err := serviceSpec("druid", root, "web", serviceSelector(refPVCName(root), "web", "web", "http", map[string]int{"http": 1}), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) + service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) if err != nil { t.Fatal(err) } diff --git a/test/mock/services.go b/test/mock/services.go index d1a27c29..24d9c242 100644 --- a/test/mock/services.go +++ b/test/mock/services.go @@ -317,18 +317,18 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) DeleteRuntime(root, purgeData } // ExpectedPorts mocks base method. -func (m *MockRuntimeBackendInterface) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, globalPorts []domain.Port, reservedPorts []domain.Port) ([]domain.RuntimePortStatus, error) { +func (m *MockRuntimeBackendInterface) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExpectedPorts", root, commands, globalPorts, reservedPorts) + ret := m.ctrl.Call(m, "ExpectedPorts", root, commands, ports) ret0, _ := ret[0].([]domain.RuntimePortStatus) ret1, _ := ret[1].(error) return ret0, ret1 } // ExpectedPorts indicates an expected call of ExpectedPorts. -func (mr *MockRuntimeBackendInterfaceMockRecorder) ExpectedPorts(root, commands, globalPorts, reservedPorts any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) ExpectedPorts(root, commands, ports any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpectedPorts", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).ExpectedPorts), root, commands, globalPorts, reservedPorts) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpectedPorts", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).ExpectedPorts), root, commands, ports) } // Name mocks base method. @@ -360,18 +360,18 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) RootRef(id, namespace any) *g } // RoutingTargets mocks base method. -func (m *MockRuntimeBackendInterface) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, globalPorts, reservedPorts []domain.Port) ([]domain.RuntimeRoutingTarget, error) { +func (m *MockRuntimeBackendInterface) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RoutingTargets", root, commands, globalPorts, reservedPorts) + ret := m.ctrl.Call(m, "RoutingTargets", root, commands, ports) ret0, _ := ret[0].([]domain.RuntimeRoutingTarget) ret1, _ := ret[1].(error) return ret0, ret1 } // RoutingTargets indicates an expected call of RoutingTargets. -func (mr *MockRuntimeBackendInterfaceMockRecorder) RoutingTargets(root, commands, globalPorts, reservedPorts any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) RoutingTargets(root, commands, ports any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RoutingTargets", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).RoutingTargets), root, commands, globalPorts, reservedPorts) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RoutingTargets", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).RoutingTargets), root, commands, ports) } // RunCommand mocks base method. From c494d8d1beb390fbb42b8579b77e0fcbdbe43793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Schottst=C3=A4dt?= Date: Sat, 22 Aug 2026 23:49:16 +0200 Subject: [PATCH 2/3] refactor(runtime)!: inject status observer Backends receive one supervisor-owned observer at creation instead of carrying callbacks on every command. StopCommand is now required by the primary backend interface. BREAKING CHANGE: runtime backend factories must create backends with a ProcedureStatusObserver, and RuntimeBackendInterface implementations must provide StopCommand. --- apps/druid/adapters/cli/daemon.go | 5 +- apps/druid/core/services/runtime_commands.go | 6 +- .../core/services/runtime_session_cache.go | 3 +- .../services/runtime_session_execution.go | 3 - .../runtime_session_execution_test.go | 20 +-- .../druid/core/services/runtime_supervisor.go | 40 ++++- .../core/services/runtime_supervisor_test.go | 139 ++++++++++++++---- apps/druid/core/services/runtime_ui_test.go | 2 +- internal/core/ports/services_ports.go | 64 +++++--- internal/runtime/backend.go | 51 +++++-- internal/runtime/backend_factory_test.go | 48 +++++- internal/runtime/docker/backend.go | 33 +++-- internal/runtime/docker/procedures.go | 35 +++-- internal/runtime/kubernetes/backend.go | 37 +++-- internal/runtime/kubernetes/procedures.go | 35 +++-- test/mock/services.go | 30 +--- 16 files changed, 379 insertions(+), 172 deletions(-) diff --git a/apps/druid/adapters/cli/daemon.go b/apps/druid/adapters/cli/daemon.go index f5883340..3c975242 100644 --- a/apps/druid/adapters/cli/daemon.go +++ b/apps/druid/adapters/cli/daemon.go @@ -145,7 +145,10 @@ func runRuntimeDaemon() error { return err } manager := services.NewRuntimeScrollManager(runtime.Store) - supervisor := appservices.NewRuntimeSupervisor(runtime.Store, manager, runtime.Backend) + supervisor, err := appservices.NewRuntimeSupervisor(runtime.Store, manager, runtime) + if err != nil { + return err + } callbacks := appservices.NewWorkerCallbackManager() supervisor.SetWorkerTimeout(runtimeWorkerTimeout) callbackConfig := ports.RuntimeWorkerCallbackConfig{ diff --git a/apps/druid/core/services/runtime_commands.go b/apps/druid/core/services/runtime_commands.go index cdca0b85..04882fa3 100644 --- a/apps/druid/core/services/runtime_commands.go +++ b/apps/druid/core/services/runtime_commands.go @@ -76,10 +76,8 @@ func (s *RuntimeSession) RemoveCommand(command string) error { s.mu.Lock() root := s.runtimeScroll.Root s.mu.Unlock() - if stopper, ok := s.runtimeBackend.(interface{ StopCommand(string, string) error }); ok { - if err := stopper.StopCommand(root, command); err != nil { - return err - } + if err := s.runtimeBackend.StopCommand(root, command); err != nil { + return err } s.mu.Lock() diff --git a/apps/druid/core/services/runtime_session_cache.go b/apps/druid/core/services/runtime_session_cache.go index 3d882a5a..3fc7a79c 100644 --- a/apps/druid/core/services/runtime_session_cache.go +++ b/apps/druid/core/services/runtime_session_cache.go @@ -50,16 +50,15 @@ func (s *RuntimeSupervisor) startSession(runtimeScroll *domain.RuntimeScroll) (* if err != nil { return nil, err } - session.Start() s.mu.Lock() if existing := s.sessions[runtimeScroll.ID]; existing != nil { s.mu.Unlock() - session.stopDeploymentQueue() return existing, nil } s.sessions[runtimeScroll.ID] = session s.mu.Unlock() + session.Start() return session, nil } diff --git a/apps/druid/core/services/runtime_session_execution.go b/apps/druid/core/services/runtime_session_execution.go index 81aa5886..e455d0b8 100644 --- a/apps/druid/core/services/runtime_session_execution.go +++ b/apps/druid/core/services/runtime_session_execution.go @@ -77,9 +77,6 @@ func (s *RuntimeSession) runCommand(cmd string, extraEnv ...map[string]map[strin Ports: runtimePorts, Routing: routing, ProcedureEnv: procedureEnv, - ProcedureStatusObserver: func(procedure string, status domain.ScrollLockStatus, exitCode *int) { - s.persistProcedureStatus(cmd, procedure, status, exitCode) - }, }) if err != nil { s.setCommandProcedureStatus(cmd, command, domain.ScrollLockStatusError, exitCode) diff --git a/apps/druid/core/services/runtime_session_execution_test.go b/apps/druid/core/services/runtime_session_execution_test.go index fe327e3a..826782f5 100644 --- a/apps/druid/core/services/runtime_session_execution_test.go +++ b/apps/druid/core/services/runtime_session_execution_test.go @@ -38,10 +38,10 @@ func TestRuntimeSessionRunCommandPassesCommandContextToRuntimeBackend(t *testing func TestRuntimeSessionPersistentCommandRemainsRunningAfterSetup(t *testing.T) { session := newRuntimeSessionExecutionTest(t, executionScrollYAML(), &fakeWorkerBackend{ - runCommand: func(command ports.RuntimeCommand) (*int, error) { - command.ObserveProcedureStatus("web", domain.ScrollLockStatusRunning, nil) - return nil, nil - }, + procedureStatusUpdates: []ports.ProcedureStatusUpdate{{ + Procedure: "web", + Status: domain.ScrollLockStatusRunning, + }}, }) session.Start() @@ -161,13 +161,12 @@ func TestRuntimeSessionRunCommandRejectsUnassignedDynamicPort(t *testing.T) { } } -func TestRuntimeSessionRunCommandPersistsProcedureStatusCallbacks(t *testing.T) { +func TestRuntimeSessionRunCommandPersistsProcedureStatusUpdates(t *testing.T) { exitCode := 0 session := newRuntimeSessionExecutionTest(t, executionScrollYAML(), &fakeWorkerBackend{ - runCommand: func(command ports.RuntimeCommand) (*int, error) { - command.ObserveProcedureStatus("web", domain.ScrollLockStatusRunning, nil) - command.ObserveProcedureStatus("web", domain.ScrollLockStatusDone, &exitCode) - return nil, nil + procedureStatusUpdates: []ports.ProcedureStatusUpdate{ + {Procedure: "web", Status: domain.ScrollLockStatusRunning}, + {Procedure: "web", Status: domain.ScrollLockStatusDone, ExitCode: &exitCode}, }, }) @@ -242,6 +241,9 @@ func newRuntimeSessionExecutionTest(t *testing.T, scrollYAML string, backend *fa if err != nil { t.Fatal(err) } + backend.procedureStatusObserver = ports.ProcedureStatusObserverFunc(func(update ports.ProcedureStatusUpdate) { + session.persistProcedureStatus(update.Command, update.Procedure, update.Status, update.ExitCode) + }) return session } diff --git a/apps/druid/core/services/runtime_supervisor.go b/apps/druid/core/services/runtime_supervisor.go index 8f4fe91d..6a63a23b 100644 --- a/apps/druid/core/services/runtime_supervisor.go +++ b/apps/druid/core/services/runtime_supervisor.go @@ -12,6 +12,8 @@ import ( "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" coreservices "github.com/highcard-dev/daemon/internal/core/services" + "github.com/highcard-dev/daemon/internal/utils/logger" + "go.uber.org/zap" ) // RuntimeSupervisor is the daemon-facing coordinator. It owns persisted runtime @@ -53,15 +55,37 @@ func fixedDeveloperPorts() []domain.Port { func NewRuntimeSupervisor( store ports.RuntimeScrollStore, manager *coreservices.RuntimeScrollManager, - runtimeBackend ports.RuntimeBackendInterface, -) *RuntimeSupervisor { - return &RuntimeSupervisor{ - store: store, - manager: manager, - runtimeBackend: runtimeBackend, - workerTimeout: 20 * time.Minute, - sessions: map[string]*RuntimeSession{}, + backendFactory ports.RuntimeBackendFactory, +) (*RuntimeSupervisor, error) { + if backendFactory == nil { + return nil, fmt.Errorf("runtime backend factory is required") } + supervisor := &RuntimeSupervisor{ + store: store, + manager: manager, + workerTimeout: 20 * time.Minute, + sessions: map[string]*RuntimeSession{}, + } + runtimeBackend, err := backendFactory.Create(supervisor) + if err != nil { + return nil, err + } + if runtimeBackend == nil { + return nil, fmt.Errorf("runtime backend factory returned nil") + } + supervisor.runtimeBackend = runtimeBackend + return supervisor, nil +} + +func (s *RuntimeSupervisor) ObserveProcedureStatus(update ports.ProcedureStatusUpdate) { + s.mu.Lock() + session := s.sessions[update.RuntimeID] + s.mu.Unlock() + if session == nil { + logger.Log().Warn("Ignoring procedure status for an unknown runtime", zap.String("runtime", update.RuntimeID), zap.String("command", update.Command), zap.String("procedure", update.Procedure), zap.String("status", string(update.Status))) + return + } + session.persistProcedureStatus(update.Command, update.Procedure, update.Status, update.ExitCode) } func (s *RuntimeSupervisor) SetWorkerCallbacks(callbacks *WorkerCallbackManager, callbackURL string) { diff --git a/apps/druid/core/services/runtime_supervisor_test.go b/apps/druid/core/services/runtime_supervisor_test.go index 71cf2b4d..fead6e74 100644 --- a/apps/druid/core/services/runtime_supervisor_test.go +++ b/apps/druid/core/services/runtime_supervisor_test.go @@ -46,6 +46,56 @@ commands: } } +func TestRuntimeSupervisorRoutesProcedureStatusUpdatesByRuntime(t *testing.T) { + store := newTestStateStore(t) + backend := &fakeWorkerBackend{} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + + for _, id := range []string{"runtime-a", "runtime-b"} { + runtimeScroll := &domain.RuntimeScroll{ + ID: id, + Artifact: "local", + Root: t.TempDir(), + ScrollName: id, + ScrollYAML: executionScrollYAML(), + Procedures: domain.ProcedureStatusMap{}, + } + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + session, err := NewRuntimeSession(store, runtimeScroll, backend) + if err != nil { + t.Fatal(err) + } + supervisor.sessions[id] = session + } + + exitCode := 17 + backend.procedureStatusObserver.ObserveProcedureStatus(ports.ProcedureStatusUpdate{ + RuntimeID: "runtime-b", + Command: "serve", + Procedure: "web", + Status: domain.ScrollLockStatusError, + ExitCode: &exitCode, + }) + + runtimeA, err := store.GetScroll("runtime-a") + if err != nil { + t.Fatal(err) + } + if len(runtimeA.Procedures) != 0 { + t.Fatalf("runtime-a procedures = %#v, want no updates", runtimeA.Procedures) + } + runtimeB, err := store.GetScroll("runtime-b") + if err != nil { + t.Fatal(err) + } + status := runtimeB.Procedures["serve"]["web"] + if status.Status != domain.ScrollLockStatusError || status.ExitCode == nil || *status.ExitCode != exitCode { + t.Fatalf("runtime-b status = %#v, want error with exit %d", status, exitCode) + } +} + func TestRuntimeSessionRepairsLegacyNilCommands(t *testing.T) { store := newTestStateStore(t) runtimeScroll := &domain.RuntimeScroll{ @@ -389,7 +439,7 @@ func TestRuntimeSupervisorStartDoesNotHydrateStoppedScroll(t *testing.T) { if err := store.CreateScroll(runtimeScroll); err != nil { t.Fatal(err) } - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{}) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{}) if err := supervisor.Start(); err != nil { t.Fatal(err) @@ -423,7 +473,7 @@ func TestRuntimeSupervisorStartHydratesRunningScroll(t *testing.T) { if err := store.CreateScroll(runtimeScroll); err != nil { t.Fatal(err) } - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{}) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{}) if err := supervisor.Start(); err != nil { t.Fatal(err) @@ -469,7 +519,7 @@ func TestRuntimeSupervisorStartRunsHydratedRestartCommand(t *testing.T) { } called := make(chan ports.RuntimeCommand, 1) release := make(chan struct{}) - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{ + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{ runCommand: func(command ports.RuntimeCommand) (*int, error) { called <- command <-release @@ -504,7 +554,7 @@ func TestRuntimeSupervisorEnsureCanCreate(t *testing.T) { } store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start")}, @@ -535,7 +585,7 @@ func TestRuntimeSupervisorEnsureDoesNotChangeReservedPortState(t *testing.T) { if err := store.CreateScroll(scroll); err != nil { t.Fatal(err) } - supervisor := NewRuntimeSupervisor(store, nil, &fakeWorkerBackend{}) + supervisor := newRuntimeSupervisorForTest(t, store, nil, &fakeWorkerBackend{}) got, err := supervisor.Ensure(EnsureOptions{Name: scroll.ID}) if err != nil { t.Fatal(err) @@ -552,7 +602,7 @@ func TestRuntimeSupervisorCreateCanCreate(t *testing.T) { } store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start")}, @@ -579,7 +629,7 @@ func TestRuntimeSupervisorCreateGeneratesIDWhenNameOmitted(t *testing.T) { store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start"), digest: "sha256:generated"} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -611,7 +661,7 @@ func TestRuntimeSupervisorCreateUsesPullWorkerBeforeStateMutation(t *testing.T) store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start"), digest: "sha256:worker"} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -643,7 +693,7 @@ func TestRuntimeSupervisorCreateWorkerFailureLeavesGeneratedPlaceholder(t *testi store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, workerErr: errors.New("pull image failed")} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -667,7 +717,7 @@ func TestRuntimeSupervisorCreateWorkerFailureLeavesGeneratedPlaceholder(t *testi func TestRuntimeSupervisorCreateRequiresWorkerCallbackConfig(t *testing.T) { store := newTestStateStore(t) - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), &fakeWorkerBackend{scrollYAML: cachedScrollYAML("start")}, @@ -689,7 +739,7 @@ func TestRuntimeSupervisorCreateUsesRequestedNamespaceForRoot(t *testing.T) { store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start")} - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") runtimeScroll, err := supervisor.CreateWithOwner("registry.local/lab:1.0", "worker-scroll", "owner-a", "games", nil) @@ -705,7 +755,7 @@ func TestRuntimeSupervisorEnsureRetriesIncompleteMaterializationFailure(t *testi store := newTestStateStore(t) callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, workerErr: errors.New("pull image failed")} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -748,7 +798,7 @@ func TestRuntimeSupervisorEnsureRepairsIncompletePlaceholder(t *testing.T) { } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: cachedScrollYAML("start"), digest: "sha256:repair"} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -784,7 +834,7 @@ func TestRuntimeSupervisorEnsureDoesNotRetryExistingError(t *testing.T) { } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: updatedScrollYAML("invalid-scroll")} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -823,7 +873,7 @@ func TestRuntimeSupervisorEnsureUpdatesChangedArtifact(t *testing.T) { } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: updatedScrollYAMLWithPorts("updated-scroll", "main")} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -879,7 +929,7 @@ func TestRuntimeSupervisorUpdateUsesPullWorkerWhenAvailable(t *testing.T) { } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: updatedScrollYAML("updated-worker"), digest: "sha256:updated"} - supervisor := NewRuntimeSupervisor( + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend, @@ -915,7 +965,7 @@ func TestRuntimeSupervisorUpdateRefreshesCurrentArtifactAndRestartsRunningScroll } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: updatedScrollYAML("refreshed-worker"), digest: "sha256:refreshed"} - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") updated, err := supervisor.Update("refresh-worker", "", nil) @@ -964,7 +1014,7 @@ func TestRuntimeSupervisorRestoreUsesPullWorkerResult(t *testing.T) { } callbacks := NewWorkerCallbackManager() backend := &fakeWorkerBackend{callbacks: callbacks, scrollYAML: updatedScrollYAMLWithPorts("restored-worker", "main"), digest: "sha256:restored"} - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") restored, err := supervisor.Restore("restore-worker", "registry.local/backup:1.0", false, []domain.RegistryCredential{{Host: "registry.local", Username: "bot"}}) @@ -1168,7 +1218,7 @@ func TestDeriveRuntimeScrollStatusTreatsDoneFiniteAsStopped(t *testing.T) { func TestDeleteDoesNotParseScrollYAML(t *testing.T) { store := newTestStateStore(t) backend := &fakeWorkerBackend{} - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) if err := store.CreateScroll(&domain.RuntimeScroll{ ID: "delete-only", Root: "runtime://delete-only", @@ -1229,16 +1279,27 @@ func newRuntimeSessionForTest(t *testing.T, commands map[string]domain.LockStatu } type fakeWorkerBackend struct { - callbacks *WorkerCallbackManager - scrollYAML string - digest string - workerErr error - action ports.RuntimeWorkerAction - stopRoot string - deleteRoot string - spawnCount int - runCommand func(ports.RuntimeCommand) (*int, error) - stopRuntime func(string) error + callbacks *WorkerCallbackManager + procedureStatusObserver ports.ProcedureStatusObserver + procedureStatusUpdates []ports.ProcedureStatusUpdate + scrollYAML string + digest string + workerErr error + action ports.RuntimeWorkerAction + stopRoot string + deleteRoot string + spawnCount int + runCommand func(ports.RuntimeCommand) (*int, error) + stopRuntime func(string) error +} + +func newRuntimeSupervisorForTest(t *testing.T, store ports.RuntimeScrollStore, manager *coreservices.RuntimeScrollManager, factory ports.RuntimeBackendFactory) *RuntimeSupervisor { + t.Helper() + supervisor, err := NewRuntimeSupervisor(store, manager, factory) + if err != nil { + t.Fatal(err) + } + return supervisor } func (f *fakeWorkerBackend) Name() string { @@ -1252,7 +1313,23 @@ func (f *fakeWorkerBackend) RootRef(id string, namespace string) string { return "runtime://" + id } +func (f *fakeWorkerBackend) Create(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + f.procedureStatusObserver = observer + return f, nil +} + func (f *fakeWorkerBackend) RunCommand(command ports.RuntimeCommand) (*int, error) { + for _, update := range f.procedureStatusUpdates { + if update.RuntimeID == "" { + update.RuntimeID = command.ScrollID + } + if update.Command == "" { + update.Command = command.Name + } + if f.procedureStatusObserver != nil { + f.procedureStatusObserver.ObserveProcedureStatus(update) + } + } if f.runCommand != nil { return f.runCommand(command) } @@ -1279,6 +1356,10 @@ func (f *fakeWorkerBackend) Signal(commandName string, target string, signal str return nil } +func (f *fakeWorkerBackend) StopCommand(root string, command string) error { + return nil +} + func (f *fakeWorkerBackend) StopRuntime(root string) error { if f.stopRuntime != nil { return f.stopRuntime(root) diff --git a/apps/druid/core/services/runtime_ui_test.go b/apps/druid/core/services/runtime_ui_test.go index a35a5b7f..08b87689 100644 --- a/apps/druid/core/services/runtime_ui_test.go +++ b/apps/druid/core/services/runtime_ui_test.go @@ -52,7 +52,7 @@ func TestPublishUIPackageRunsOneShotCommandWithEphemeralURL(t *testing.T) { command = got return nil, nil }} - supervisor := NewRuntimeSupervisor(store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) updated, err := supervisor.PublishUIPackage("ui-scroll", "private", "") if err != nil { diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index 27a3fbcc..ae5d62b5 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -42,6 +42,7 @@ type RuntimeBackendInterface interface { CreateUIPackageUpload(ctx context.Context, action RuntimeUIPackageUploadAction) (RuntimeUIPackageUpload, error) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) + StopCommand(root string, command string) error StopRuntime(root string) error DeleteRuntime(root string, purgeData bool) error BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error @@ -50,10 +51,16 @@ type RuntimeBackendInterface interface { Signal(commandName string, target string, signal string, root string) error } -// RuntimeCommandStopper is implemented by backends that can remove one -// command's workloads without stopping the rest of the runtime. -type RuntimeCommandStopper interface { - StopCommand(root string, command string) error +// RuntimeBackendFactory creates a backend with its required status observer. +type RuntimeBackendFactory interface { + Create(observer ProcedureStatusObserver) (RuntimeBackendInterface, error) +} + +// RuntimeBackendFactoryFunc adapts a function to RuntimeBackendFactory. +type RuntimeBackendFactoryFunc func(observer ProcedureStatusObserver) (RuntimeBackendInterface, error) + +func (f RuntimeBackendFactoryFunc) Create(observer ProcedureStatusObserver) (RuntimeBackendInterface, error) { + return f(observer) } type RuntimeWorkerCallbackConfig struct { @@ -92,22 +99,41 @@ type RuntimeScrollStore interface { } type RuntimeCommand struct { - Name string + // Name identifies the command within the Scroll. + Name string + // ScrollID identifies the runtime Scroll whose command is being executed. ScrollID string - Command *domain.CommandInstructionSet - Root string - // Ports is the complete runtime port set after reserved ports are merged - // and dynamic ports are resolved from routing assignments. - Ports []domain.Port - Routing []domain.RuntimeRouteAssignment - ProcedureEnv map[string]map[string]string - ProcedureStatusObserver func(procedure string, status domain.ScrollLockStatus, exitCode *int) -} - -func (c RuntimeCommand) ObserveProcedureStatus(procedure string, status domain.ScrollLockStatus, exitCode *int) { - if c.ProcedureStatusObserver != nil { - c.ProcedureStatusObserver(procedure, status, exitCode) - } + // Command contains the procedures and run mode to execute. + Command *domain.CommandInstructionSet + // Root identifies the backend-owned runtime data location. + Root string + // Ports contains all runtime ports after merging reservations and resolving dynamic assignments. + Ports []domain.Port + // Routing contains the runtime's external route assignments used by backends during execution. + Routing []domain.RuntimeRouteAssignment + // ProcedureEnv contains the fully resolved environment for each procedure name. + ProcedureEnv map[string]map[string]string +} + +// ProcedureStatusUpdate identifies one backend-reported procedure transition. +type ProcedureStatusUpdate struct { + RuntimeID string + Command string + Procedure string + Status domain.ScrollLockStatus + ExitCode *int +} + +// ProcedureStatusObserver receives procedure transitions from a runtime backend. +type ProcedureStatusObserver interface { + ObserveProcedureStatus(update ProcedureStatusUpdate) +} + +// ProcedureStatusObserverFunc adapts a function to ProcedureStatusObserver. +type ProcedureStatusObserverFunc func(update ProcedureStatusUpdate) + +func (f ProcedureStatusObserverFunc) ObserveProcedureStatus(update ProcedureStatusUpdate) { + f(update) } type RuntimeUIPackageUploadAction struct { diff --git a/internal/runtime/backend.go b/internal/runtime/backend.go index e5b4ea9c..90439c4c 100644 --- a/internal/runtime/backend.go +++ b/internal/runtime/backend.go @@ -11,6 +11,7 @@ import ( type Runtime struct { Backend ports.RuntimeBackendInterface Store ports.RuntimeScrollStore + create ports.RuntimeBackendFactoryFunc } type Options struct { @@ -32,12 +33,12 @@ func WithDockerConfig(config docker.Config) Option { } } -var newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface) (ports.RuntimeBackendInterface, error) { - return docker.NewWithConfig(config, consoleManager) +var newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return docker.NewWithConfig(config, consoleManager, observer) } -var newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface) (ports.RuntimeBackendInterface, error) { - return runtimekubernetes.New(config, consoleManager) +var newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return runtimekubernetes.New(config, consoleManager, observer) } var newKubernetesStateStore = func(config runtimekubernetes.Config) (ports.RuntimeScrollStore, error) { @@ -51,33 +52,53 @@ func NewRuntime(name string, consoleManager ports.ConsoleManagerInterface, state } switch name { case "", "docker": - backend, err := newDockerBackend(options.Docker, consoleManager) - if err != nil { - return nil, err - } store, err := docker.NewStateStore(stateDir) if err != nil { return nil, err } return &Runtime{ - Backend: backend, - Store: dockerRuntimeStore{RuntimeScrollStore: store, config: options.Docker.WithDefaults()}, + Store: dockerRuntimeStore{RuntimeScrollStore: store, config: options.Docker.WithDefaults()}, + create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return newDockerBackend(options.Docker, consoleManager, observer) + }, }, nil case "kubernetes": - backend, err := newKubernetesBackend(options.Kubernetes, consoleManager) - if err != nil { - return nil, err - } store, err := newKubernetesStateStore(options.Kubernetes) if err != nil { return nil, err } - return &Runtime{Backend: backend, Store: store}, nil + return &Runtime{ + Store: store, + create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return newKubernetesBackend(options.Kubernetes, consoleManager, observer) + }, + }, nil default: return nil, fmt.Errorf("unknown runtime backend %q", name) } } +func (r *Runtime) Create(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + if observer == nil { + return nil, fmt.Errorf("procedure status observer is required") + } + if r.Backend != nil { + return nil, fmt.Errorf("runtime backend already created") + } + if r.create == nil { + return nil, fmt.Errorf("runtime backend factory is not configured") + } + backend, err := r.create(observer) + if err != nil { + return nil, err + } + if backend == nil { + return nil, fmt.Errorf("runtime backend factory returned nil") + } + r.Backend = backend + return backend, nil +} + type dockerRuntimeStore struct { ports.RuntimeScrollStore config docker.Config diff --git a/internal/runtime/backend_factory_test.go b/internal/runtime/backend_factory_test.go index 36d9e0c0..1c3f5cc3 100644 --- a/internal/runtime/backend_factory_test.go +++ b/internal/runtime/backend_factory_test.go @@ -15,10 +15,13 @@ import ( func TestNewRuntimeDockerOwnsStoreSelection(t *testing.T) { previousDocker := newDockerBackend - newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface) (ports.RuntimeBackendInterface, error) { + newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { if config.VolumePrefix != "lab" { t.Fatalf("volume prefix = %s, want lab", config.VolumePrefix) } + if observer == nil { + t.Fatal("procedure status observer is nil") + } return fakeBackend{name: "docker"}, nil } t.Cleanup(func() { newDockerBackend = previousDocker }) @@ -27,6 +30,9 @@ func TestNewRuntimeDockerOwnsStoreSelection(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := runtime.Create(ports.ProcedureStatusObserverFunc(func(ports.ProcedureStatusUpdate) {})); err != nil { + t.Fatal(err) + } if runtime.Backend.Name() != "docker" { t.Fatalf("backend = %s, want docker", runtime.Backend.Name()) } @@ -39,10 +45,13 @@ func TestNewRuntimeKubernetesOwnsStoreSelection(t *testing.T) { stateDir := t.TempDir() previousBackend := newKubernetesBackend previousStore := newKubernetesStateStore - newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface) (ports.RuntimeBackendInterface, error) { + newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { if config.Namespace != "druid" { t.Fatalf("backend namespace = %s, want druid", config.Namespace) } + if observer == nil { + t.Fatal("procedure status observer is nil") + } return fakeBackend{name: "kubernetes"}, nil } newKubernetesStateStore = func(config runtimekubernetes.Config) (ports.RuntimeScrollStore, error) { @@ -60,6 +69,9 @@ func TestNewRuntimeKubernetesOwnsStoreSelection(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := runtime.Create(ports.ProcedureStatusObserverFunc(func(ports.ProcedureStatusUpdate) {})); err != nil { + t.Fatal(err) + } if runtime.Backend.Name() != "kubernetes" { t.Fatalf("backend = %s, want kubernetes", runtime.Backend.Name()) } @@ -78,6 +90,34 @@ func TestNewRuntimeUnknownBackendErrorsOnce(t *testing.T) { } } +func TestRuntimeCreateRequiresObserver(t *testing.T) { + called := false + runtime := &Runtime{create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + called = true + return fakeBackend{name: "docker"}, nil + }} + + if _, err := runtime.Create(nil); err == nil || !strings.Contains(err.Error(), "observer is required") { + t.Fatalf("Create error = %v, want required observer", err) + } + if called { + t.Fatal("backend creator called without an observer") + } +} + +func TestRuntimeCreateOnlyCreatesBackendOnce(t *testing.T) { + runtime := &Runtime{create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return fakeBackend{name: "docker"}, nil + }} + observer := ports.ProcedureStatusObserverFunc(func(ports.ProcedureStatusUpdate) {}) + if _, err := runtime.Create(observer); err != nil { + t.Fatal(err) + } + if _, err := runtime.Create(observer); err == nil || !strings.Contains(err.Error(), "already created") { + t.Fatalf("second Create error = %v, want already created", err) + } +} + type fakeBackend struct { name string } @@ -109,6 +149,10 @@ func (f fakeBackend) RoutingTargets(root string, commands map[string]*domain.Com return nil, nil } +func (f fakeBackend) StopCommand(root string, command string) error { + return nil +} + func (f fakeBackend) StopRuntime(root string) error { return nil } diff --git a/internal/runtime/docker/backend.go b/internal/runtime/docker/backend.go index 6c046290..525c2cdb 100644 --- a/internal/runtime/docker/backend.go +++ b/internal/runtime/docker/backend.go @@ -12,12 +12,13 @@ import ( ) type Backend struct { - client *client.Client - consoleManager ports.ConsoleManagerInterface - config Config - mu sync.Mutex - containers map[string]string - stdin map[string]io.Writer + client *client.Client + consoleManager ports.ConsoleManagerInterface + procedureStatusObserver ports.ProcedureStatusObserver + config Config + mu sync.Mutex + containers map[string]string + stdin map[string]io.Writer } type Config struct { @@ -92,11 +93,14 @@ func (c Config) ValidateForUIPublishing() error { return nil } -func New(consoleManager ports.ConsoleManagerInterface) (*Backend, error) { - return NewWithConfig(Config{}, consoleManager) +func New(consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { + return NewWithConfig(Config{}, consoleManager, observer) } -func NewWithConfig(config Config, consoleManager ports.ConsoleManagerInterface) (*Backend, error) { +func NewWithConfig(config Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { + if observer == nil { + return nil, fmt.Errorf("procedure status observer is required") + } config = config.WithDefaults() if config.Storage != StorageVolume && config.Storage != StorageBind { return nil, fmt.Errorf("unknown docker storage %q", config.Storage) @@ -114,11 +118,12 @@ func NewWithConfig(config Config, consoleManager ports.ConsoleManagerInterface) return nil, err } return &Backend{ - client: cli, - consoleManager: consoleManager, - config: config, - containers: map[string]string{}, - stdin: map[string]io.Writer{}, + client: cli, + consoleManager: consoleManager, + procedureStatusObserver: observer, + config: config, + containers: map[string]string{}, + stdin: map[string]io.Writer{}, }, nil } diff --git a/internal/runtime/docker/procedures.go b/internal/runtime/docker/procedures.go index b964e840..fb2ecee6 100644 --- a/internal/runtime/docker/procedures.go +++ b/internal/runtime/docker/procedures.go @@ -34,47 +34,60 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { ) if command.Command.Run == domain.RunModePersistent { if procedure.IsSignal() { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) if err := b.Signal(procedureName, procedure.Target, procedure.Signal, command.Root); err != nil { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) return nil, err } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, nil) continue } if procedure.Image == "" { return nil, fmt.Errorf("docker runtime procedure %s requires image", procedureName) } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) if err := b.startPersistentContainer(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env); err != nil { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) return nil, err } continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) exitCode, err := b.runProcedure(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env) if err != nil { if exitCode != nil && *exitCode != 0 && procedure.IgnoreFailure { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, exitCode) return exitCode, err } if exitCode != nil && *exitCode != 0 { if procedure.IgnoreFailure { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, exitCode) return exitCode, nil } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) } return nil, nil } +func (b *Backend) observeProcedureStatus(command ports.RuntimeCommand, procedure string, status domain.ScrollLockStatus, exitCode *int) { + if b.procedureStatusObserver == nil { + return + } + b.procedureStatusObserver.ObserveProcedureStatus(ports.ProcedureStatusUpdate{ + RuntimeID: command.ScrollID, + Command: command.Name, + Procedure: procedure, + Status: status, + ExitCode: exitCode, + }) +} + func (b *Backend) runProcedure(consoleID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) (*int, error) { if procedure.IsSignal() { return nil, b.Signal(procedureName, procedure.Target, procedure.Signal, root) diff --git a/internal/runtime/kubernetes/backend.go b/internal/runtime/kubernetes/backend.go index b649347b..8480ce5f 100644 --- a/internal/runtime/kubernetes/backend.go +++ b/internal/runtime/kubernetes/backend.go @@ -18,15 +18,16 @@ import ( ) type Backend struct { - client k8sclient.Interface - restConfig *rest.Config - httpClient *http.Client - consoleManager ports.ConsoleManagerInterface - config Config - statsReader nodeStatsReader - jobLogRunner func(context.Context, *batchv1.Job) ([]byte, error) - jobExitMu sync.Mutex - jobExits map[string]recentJobExit + client k8sclient.Interface + restConfig *rest.Config + httpClient *http.Client + consoleManager ports.ConsoleManagerInterface + procedureStatusObserver ports.ProcedureStatusObserver + config Config + statsReader nodeStatsReader + jobLogRunner func(context.Context, *batchv1.Job) ([]byte, error) + jobExitMu sync.Mutex + jobExits map[string]recentJobExit } type recentJobExit struct { @@ -39,7 +40,10 @@ const ( defaultKubernetesClientBurst int = 100 ) -func New(config Config, consoleManager ports.ConsoleManagerInterface) (*Backend, error) { +func New(config Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { + if observer == nil { + return nil, fmt.Errorf("procedure status observer is required") + } config = config.WithDefaults() restConfig, namespace, source, _, err := runtimeRESTConfig(config) @@ -64,12 +68,13 @@ func New(config Config, consoleManager ports.ConsoleManagerInterface) (*Backend, } logger.Log().Info("Using Kubernetes backend settings", zap.String("source", source), zap.String("namespace", config.Namespace)) backend := &Backend{ - client: client, - restConfig: restConfig, - httpClient: httpClient, - consoleManager: consoleManager, - config: config, - jobExits: make(map[string]recentJobExit), + client: client, + restConfig: restConfig, + httpClient: httpClient, + consoleManager: consoleManager, + procedureStatusObserver: observer, + config: config, + jobExits: make(map[string]recentJobExit), } backend.statsReader = backend.readNodeStatsSummary if config.PullImage == "" { diff --git a/internal/runtime/kubernetes/procedures.go b/internal/runtime/kubernetes/procedures.go index d1eb6ffc..c601d426 100644 --- a/internal/runtime/kubernetes/procedures.go +++ b/internal/runtime/kubernetes/procedures.go @@ -83,13 +83,13 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { ) if command.Command.Run == domain.RunModePersistent { if procedure.IsSignal() { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) if err := b.Signal(procedureName, procedure.Target, procedure.Signal, command.Root); err != nil { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) logger.Log().Error("Kubernetes signal procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal), zap.Error(err)) return nil, err } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, nil) logger.Log().Info("Kubernetes signal procedure completed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal)) continue } @@ -98,37 +98,37 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { logger.Log().Error("Kubernetes persistent procedure missing image", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Error(err)) return nil, err } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) if err := b.ensurePersistentProcedure(context.Background(), command.ScrollID, command.Root, command.Name, procedureName, resourceName, procedure, command.Ports, env); err != nil { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) logger.Log().Error("Kubernetes persistent procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Error(err)) return nil, err } continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusRunning, nil) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) exitCode, err := b.runJobProcedure(command.ScrollID, command.Name, procedureName, resourceName, procedure, command.Root, command.Ports, env) if err != nil { if exitCode != nil && *exitCode != 0 && procedure.IgnoreFailure { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) logger.Log().Warn("Kubernetes job procedure failed but failure is ignored", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Int("exit_code", *exitCode), zap.Error(err)) continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, exitCode) logger.Log().Error("Kubernetes job procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Any("exit_code", exitCode), zap.Error(err)) return exitCode, err } if exitCode != nil && *exitCode != 0 { if procedure.IgnoreFailure { - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) logger.Log().Warn("Kubernetes job procedure failed but failure is ignored", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Int("exit_code", *exitCode)) continue } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusError, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, exitCode) logger.Log().Warn("Kubernetes command stopped after non-zero procedure exit", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Int("exit_code", *exitCode)) return exitCode, nil } - command.ObserveProcedureStatus(procedureName, domain.ScrollLockStatusDone, exitCode) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) if exitCode != nil { logger.Log().Info("Kubernetes job procedure completed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Int("exit_code", *exitCode)) } @@ -137,6 +137,19 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { return nil, nil } +func (b *Backend) observeProcedureStatus(command ports.RuntimeCommand, procedure string, status domain.ScrollLockStatus, exitCode *int) { + if b.procedureStatusObserver == nil { + return + } + b.procedureStatusObserver.ObserveProcedureStatus(ports.ProcedureStatusUpdate{ + RuntimeID: command.ScrollID, + Command: command.Name, + Procedure: procedure, + Status: status, + ExitCode: exitCode, + }) +} + func (b *Backend) runJobProcedure(scrollID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, ports []domain.Port, env map[string]string) (*int, error) { if procedure.IsSignal() { logger.Log().Info("Running Kubernetes signal procedure", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal)) diff --git a/test/mock/services.go b/test/mock/services.go index 24d9c242..0546777c 100644 --- a/test/mock/services.go +++ b/test/mock/services.go @@ -431,32 +431,8 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) StopRuntime(root any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopRuntime", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).StopRuntime), root) } -// MockRuntimeCommandStopper is a mock of RuntimeCommandStopper interface. -type MockRuntimeCommandStopper struct { - ctrl *gomock.Controller - recorder *MockRuntimeCommandStopperMockRecorder - isgomock struct{} -} - -// MockRuntimeCommandStopperMockRecorder is the mock recorder for MockRuntimeCommandStopper. -type MockRuntimeCommandStopperMockRecorder struct { - mock *MockRuntimeCommandStopper -} - -// NewMockRuntimeCommandStopper creates a new mock instance. -func NewMockRuntimeCommandStopper(ctrl *gomock.Controller) *MockRuntimeCommandStopper { - mock := &MockRuntimeCommandStopper{ctrl: ctrl} - mock.recorder = &MockRuntimeCommandStopperMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRuntimeCommandStopper) EXPECT() *MockRuntimeCommandStopperMockRecorder { - return m.recorder -} - // StopCommand mocks base method. -func (m *MockRuntimeCommandStopper) StopCommand(root, command string) error { +func (m *MockRuntimeBackendInterface) StopCommand(root, command string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "StopCommand", root, command) ret0, _ := ret[0].(error) @@ -464,9 +440,9 @@ func (m *MockRuntimeCommandStopper) StopCommand(root, command string) error { } // StopCommand indicates an expected call of StopCommand. -func (mr *MockRuntimeCommandStopperMockRecorder) StopCommand(root, command any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) StopCommand(root, command any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopCommand", reflect.TypeOf((*MockRuntimeCommandStopper)(nil).StopCommand), root, command) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopCommand", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).StopCommand), root, command) } // MockRuntimeWorkerCallbackBackend is a mock of RuntimeWorkerCallbackBackend interface. From 12d4086c3d728504e6e11db333aa9d4bf876edbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Schottst=C3=A4dt?= Date: Sun, 23 Aug 2026 12:33:32 +0200 Subject: [PATCH 3/3] refactor(runtime): unify workload execution Stream consoles directly from runtime backends and group Kubernetes persistent procedures into one StatefulSet with ordered init containers. --- api/openapi.yaml | 27 -- apps/druid/adapters/cli/daemon.go | 8 +- apps/druid/adapters/http/handlers/routes.go | 2 - .../adapters/http/handlers/scroll_handler.go | 69 +---- .../http/handlers/websocket_handler.go | 60 ++-- apps/druid/core/services/runtime_consoles.go | 56 ++++ .../core/services/runtime_materialization.go | 43 +-- .../core/services/runtime_supervisor_test.go | 120 +++++++- examples/container-lab/scroll.yaml | 36 ++- internal/api/generated.go | 230 +++------------ internal/core/domain/console.go | 13 - internal/core/domain/log.go | 10 - internal/core/ports/services_ports.go | 21 +- internal/core/services/console_manager.go | 60 ---- internal/core/services/log_manager.go | 78 ----- internal/devapi/generated.go | 42 +-- internal/runtime/backend.go | 14 +- internal/runtime/backend_factory_test.go | 21 +- internal/runtime/docker/backend.go | 11 +- internal/runtime/docker/console.go | 128 +++++--- internal/runtime/docker/console_test.go | 26 ++ internal/runtime/docker/lifecycle.go | 4 - internal/runtime/docker/names.go | 7 - internal/runtime/docker/procedures.go | 98 +------ internal/runtime/docker/workers.go | 59 ++-- internal/runtime/kubernetes/attach_signal.go | 152 +++++++--- .../runtime/kubernetes/attach_signal_test.go | 63 ++++ internal/runtime/kubernetes/backend.go | 10 +- internal/runtime/kubernetes/ports.go | 16 +- .../runtime/kubernetes/procedure_attempts.go | 9 +- internal/runtime/kubernetes/procedures.go | 161 ++++------ internal/runtime/kubernetes/resources.go | 100 ++++++- internal/runtime/kubernetes/resources_test.go | 211 +++++++++---- internal/runtime/kubernetes/wait_jobs.go | 87 +----- internal/runtime/kubernetes/workers.go | 97 +++--- .../kubernetes/workload_identity_test.go | 10 +- test/integration/example_test.go | 22 -- test/mock/services.go | 277 ++++++------------ 38 files changed, 1181 insertions(+), 1277 deletions(-) create mode 100644 apps/druid/core/services/runtime_consoles.go delete mode 100644 internal/core/domain/log.go delete mode 100644 internal/core/services/console_manager.go delete mode 100644 internal/core/services/log_manager.go create mode 100644 internal/runtime/docker/console_test.go create mode 100644 internal/runtime/kubernetes/attach_signal_test.go diff --git a/api/openapi.yaml b/api/openapi.yaml index 831e580b..32be8a07 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -14,8 +14,6 @@ servers: tags: - name: scroll description: Scroll management and command execution - - name: logs - description: Log streaming and retrieval - name: queue description: Command queue management - name: websocket @@ -285,12 +283,6 @@ components: additionalProperties: $ref: '#/components/schemas/LockStatus' - ScrollLogMap: - type: object - additionalProperties: - type: array - items: - type: string RuntimeScroll: type: object required: @@ -646,25 +638,6 @@ paths: type: object additionalProperties: true - /api/v1/scrolls/{id}/logs: - get: - operationId: getScrollLogs - summary: Get scroll-scoped logs - tags: [logs, runtime] - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - '200': - description: Logs keyed by procedure - content: - application/json: - schema: - $ref: '#/components/schemas/ScrollLogMap' - /api/v1/scrolls/{id}/ports: get: operationId: getScrollPorts diff --git a/apps/druid/adapters/cli/daemon.go b/apps/druid/adapters/cli/daemon.go index 3c975242..7209c1af 100644 --- a/apps/druid/adapters/cli/daemon.go +++ b/apps/druid/adapters/cli/daemon.go @@ -138,9 +138,7 @@ func runRuntimeDaemon() error { OperatorServiceAccount: k8sOperatorServiceAccount, } dockerConfig := runtimedocker.Config{WorkerImage: dockerWorkerImage, Storage: dockerStorage, BindRoot: dockerBindRoot, VolumePrefix: dockerVolumePrefix, UIS3Bucket: dockerUIS3Bucket, UIS3PublicBaseURL: dockerUIS3PublicBaseURL, UIS3Region: dockerUIS3Region, UIS3Endpoint: dockerUIS3Endpoint, UIS3Prefix: dockerUIS3Prefix, UIS3AccessKey: dockerUIS3AccessKey, UIS3SecretKey: dockerUIS3SecretKey, UIS3SessionToken: dockerUIS3SessionToken} - logManager := services.NewLogManager() - consoleService := services.NewConsoleManager(logManager) - runtime, err := runtimebackend.NewRuntime(runtimeBackendName, consoleService, runtimeStateDir, runtimebackend.WithKubernetesConfig(kubernetesConfig), runtimebackend.WithDockerConfig(dockerConfig)) + runtime, err := runtimebackend.NewRuntime(runtimeBackendName, runtimeStateDir, runtimebackend.WithKubernetesConfig(kubernetesConfig), runtimebackend.WithDockerConfig(dockerConfig)) if err != nil { return err } @@ -184,9 +182,9 @@ func runRuntimeDaemon() error { if err != nil { return err } - scrollHandler := runtimehandlers.NewScrollHandler(supervisor, consoleService, logManager, authorizer) + scrollHandler := runtimehandlers.NewScrollHandler(supervisor, authorizer) scrollHandler.SetAllowUnauthenticatedPublic(runtimeAllowUnauthenticatedPublic) - websocketHandler := runtimehandlers.NewWebsocketHandler(consoleService) + websocketHandler := runtimehandlers.NewWebsocketHandler() websocketHandler.SetScrollHandler(scrollHandler) websocketHandler.SetAuthorizer(authorizer) websocketHandler.SetAllowUnauthenticatedPublic(runtimeAllowUnauthenticatedPublic) diff --git a/apps/druid/adapters/http/handlers/routes.go b/apps/druid/adapters/http/handlers/routes.go index 2d5bc83b..4e4a0724 100644 --- a/apps/druid/adapters/http/handlers/routes.go +++ b/apps/druid/adapters/http/handlers/routes.go @@ -68,8 +68,6 @@ func RegisterPublicRoutes(app *fiber.App, handlers RouteHandlers) { app.Post("/:id/api/v1/command", handlers.Server.RunDaemonCommand) app.Get("/:id/api/v1/queue", handlers.Server.GetDaemonQueue) app.Get("/:id/api/v1/consoles", handlers.Server.GetDaemonConsoles) - app.Get("/:id/api/v1/logs", handlers.Server.GetDaemonLogs) - app.Get("/:id/api/v1/logs/:stream", handlers.Server.GetDaemonStreamLogs) app.Get("/:id/api/v1/ports", handlers.Server.GetDaemonPorts) app.Get("/:id/api/v1/ui/packages", handlers.Server.GetDaemonUIPackages) app.Post("/:id/api/v1/ui/packages/:scope/publish", handlers.Server.PublishDaemonUIPackage) diff --git a/apps/druid/adapters/http/handlers/scroll_handler.go b/apps/druid/adapters/http/handlers/scroll_handler.go index 9f559308..68fc4cc8 100644 --- a/apps/druid/adapters/http/handlers/scroll_handler.go +++ b/apps/druid/adapters/http/handlers/scroll_handler.go @@ -2,34 +2,28 @@ package handlers import ( "errors" - "strings" "github.com/gofiber/fiber/v2" appservices "github.com/highcard-dev/daemon/apps/druid/core/services" "github.com/highcard-dev/daemon/internal/api" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" - "github.com/highcard-dev/daemon/internal/core/services" ) type ScrollHandler struct { supervisor *appservices.RuntimeSupervisor - consoleService *services.ConsoleManager - logService *services.LogManager authorizer ports.AuthorizerServiceInterface allowUnauthenticatedPublic bool } -func NewScrollHandler(supervisor *appservices.RuntimeSupervisor, consoleService *services.ConsoleManager, logService *services.LogManager, authorizer ...ports.AuthorizerServiceInterface) *ScrollHandler { +func NewScrollHandler(supervisor *appservices.RuntimeSupervisor, authorizer ...ports.AuthorizerServiceInterface) *ScrollHandler { var auth ports.AuthorizerServiceInterface if len(authorizer) > 0 { auth = authorizer[0] } return &ScrollHandler{ - supervisor: supervisor, - consoleService: consoleService, - logService: logService, - authorizer: auth, + supervisor: supervisor, + authorizer: auth, } } @@ -232,22 +226,11 @@ func (h *ScrollHandler) GetScrollConsoles(c *fiber.Ctx, id string) error { if _, err := h.getScroll(id); err != nil { return err } - prefix := id + "/" - consoles := map[string]*domain.Console{} - for consoleID, console := range h.consoleService.GetConsoles() { - if strings.HasPrefix(consoleID, prefix) { - consoles[strings.TrimPrefix(consoleID, prefix)] = console - } - } - return c.JSON(consoles) -} - -func (h *ScrollHandler) GetScrollLogs(c *fiber.Ctx, id string) error { - logs, err := h.scrollLogs(id) + consoles, err := h.supervisor.Consoles(id) if err != nil { return err } - return c.JSON(logs) + return c.JSON(consoles) } func (h *ScrollHandler) GetDaemonScroll(c *fiber.Ctx) error { @@ -285,27 +268,6 @@ func (h *ScrollHandler) GetDaemonConsoles(c *fiber.Ctx) error { return h.GetScrollConsoles(c, c.Params("id")) } -func (h *ScrollHandler) GetDaemonLogs(c *fiber.Ctx) error { - logs, err := h.scrollLogs(c.Params("id")) - if err != nil { - return err - } - streams := make([]map[string]any, 0, len(logs)) - for stream, log := range logs { - streams = append(streams, map[string]any{"stream": stream, "log": log}) - } - return c.JSON(streams) -} - -func (h *ScrollHandler) GetDaemonStreamLogs(c *fiber.Ctx) error { - logs, err := h.scrollLogs(c.Params("id")) - if err != nil { - return err - } - stream := c.Params("stream") - return c.JSON(map[string]any{"stream": stream, "log": logs[stream]}) -} - func (h *ScrollHandler) GetDaemonPorts(c *fiber.Ctx) error { return h.GetScrollPorts(c, c.Params("id")) } @@ -436,24 +398,3 @@ func (h *ScrollHandler) getScroll(id string) (*domain.RuntimeScroll, error) { } return runtimeScroll, err } - -func (h *ScrollHandler) scrollLogs(id string) (map[string][]string, error) { - if _, err := h.getScroll(id); err != nil { - return nil, err - } - prefix := id + "/" - logs := map[string][]string{} - for streamID, log := range h.logService.GetStreams() { - if !strings.HasPrefix(streamID, prefix) { - continue - } - response := make(chan []byte, 100) - log.Req <- response - lines := []string{} - for line := range response { - lines = append(lines, string(line)) - } - logs[strings.TrimPrefix(streamID, prefix)] = lines - } - return logs, nil -} diff --git a/apps/druid/adapters/http/handlers/websocket_handler.go b/apps/druid/adapters/http/handlers/websocket_handler.go index 65dcc01f..d542d839 100644 --- a/apps/druid/adapters/http/handlers/websocket_handler.go +++ b/apps/druid/adapters/http/handlers/websocket_handler.go @@ -1,24 +1,24 @@ package handlers import ( + "context" + "io" "time" "github.com/gofiber/contrib/websocket" "github.com/highcard-dev/daemon/internal/core/ports" - "github.com/highcard-dev/daemon/internal/core/services" "github.com/highcard-dev/daemon/internal/utils/logger" "go.uber.org/zap" ) type WebsocketHandler struct { - consoleService *services.ConsoleManager scrolls *ScrollHandler authorizer ports.AuthorizerServiceInterface allowUnauthenticatedPublic bool } -func NewWebsocketHandler(consoleService *services.ConsoleManager) *WebsocketHandler { - return &WebsocketHandler{consoleService: consoleService} +func NewWebsocketHandler() *WebsocketHandler { + return &WebsocketHandler{} } func (h *WebsocketHandler) SetScrollHandler(scrolls *ScrollHandler) { @@ -34,11 +34,7 @@ func (h *WebsocketHandler) SetAllowUnauthenticatedPublic(allow bool) { } func (h *WebsocketHandler) AttachConsole(c *websocket.Conn) { - consoleID := c.Params("console") - if id := c.Params("id"); id != "" { - consoleID = id + "/" + consoleID - } - h.attach(c, consoleID) + h.attach(c, c.Params("id"), c.Params("console")) } func (h *WebsocketHandler) AttachScrollConsole(c *websocket.Conn) { @@ -49,17 +45,16 @@ func (h *WebsocketHandler) AttachScrollConsole(c *websocket.Conn) { h.AttachConsole(c) } -func (h *WebsocketHandler) attach(c *websocket.Conn, consoleID string) { +func (h *WebsocketHandler) attach(c *websocket.Conn, runtimeID string, consoleID string) { defer c.Close() - - console := h.consoleService.GetConsole(consoleID) - if console == nil { - logger.Log().Warn("Console not found", zap.String("console", consoleID)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + session, err := h.scrolls.supervisor.OpenConsole(ctx, runtimeID, consoleID) + if err != nil { + logger.Log().Warn("Console unavailable", zap.String("runtime", runtimeID), zap.String("console", consoleID), zap.Error(err)) return } - - subscription := console.Channel.Subscribe() - defer console.Channel.Unsubscribe(subscription) + defer session.Close() done := make(chan struct{}) go func() { @@ -69,12 +64,31 @@ func (h *WebsocketHandler) attach(c *websocket.Conn, consoleID string) { if err != nil { return } - if console.WriteInput != nil { - if err := console.WriteInput(string(data)); err != nil { - logger.Log().Debug("Failed to write console input", zap.Error(err)) + if _, err := session.Write(data); err != nil { + logger.Log().Debug("Failed to write console input", zap.Error(err)) + return + } + } + }() + output := make(chan []byte, 25) + go func() { + defer close(output) + buffer := make([]byte, 32*1024) + for { + n, err := session.Read(buffer) + if n > 0 { + select { + case output <- append([]byte(nil), buffer[:n]...): + case <-ctx.Done(): return } } + if err != nil { + if err != io.EOF && ctx.Err() == nil { + logger.Log().Debug("Console output ended", zap.Error(err)) + } + return + } } }() @@ -84,11 +98,11 @@ func (h *WebsocketHandler) attach(c *websocket.Conn, consoleID string) { select { case <-done: return - case data, ok := <-subscription: - if !ok || data == nil { + case data, ok := <-output: + if !ok { return } - if err := c.WriteMessage(websocket.TextMessage, *data); err != nil { + if err := c.WriteMessage(websocket.TextMessage, data); err != nil { return } case <-pingTicker.C: diff --git a/apps/druid/core/services/runtime_consoles.go b/apps/druid/core/services/runtime_consoles.go new file mode 100644 index 00000000..5a968450 --- /dev/null +++ b/apps/druid/core/services/runtime_consoles.go @@ -0,0 +1,56 @@ +package services + +import ( + "context" + "fmt" + "io" + + "github.com/highcard-dev/daemon/internal/core/domain" +) + +func (s *RuntimeSupervisor) Consoles(id string) (map[string]domain.Console, error) { + session, err := s.sessionFor(id) + if err != nil { + return nil, err + } + return session.consoles(), nil +} + +func (s *RuntimeSupervisor) OpenConsole(ctx context.Context, id string, procedure string) (io.ReadWriteCloser, error) { + session, err := s.sessionFor(id) + if err != nil { + return nil, err + } + if _, ok := session.consoles()[procedure]; !ok { + return nil, fmt.Errorf("console %s is unavailable", procedure) + } + session.mu.Lock() + root := session.runtimeScroll.Root + session.mu.Unlock() + return session.runtimeBackend.OpenConsole(ctx, root, procedure) +} + +func (s *RuntimeSession) consoles() map[string]domain.Console { + s.mu.Lock() + statuses := copyProcedureStatuses(s.runtimeScroll.Procedures) + s.mu.Unlock() + consoles := map[string]domain.Console{} + for commandName, command := range s.scrollService.GetFile().Commands { + if command == nil { + continue + } + for index, procedure := range command.Procedures { + name := domain.ProcedureName(commandName, index, procedure) + status, ok := statuses[commandName][name] + if !ok || procedure == nil || !procedure.IsContainer() { + continue + } + consoleType := domain.ConsoleTypeContainer + if procedure.TTY { + consoleType = domain.ConsoleTypeTTY + } + consoles[name] = domain.Console{Type: consoleType, InputMode: "stdin", Exit: status.ExitCode} + } + } + return consoles +} diff --git a/apps/druid/core/services/runtime_materialization.go b/apps/druid/core/services/runtime_materialization.go index f4a3e7d5..9db9ce7f 100644 --- a/apps/druid/core/services/runtime_materialization.go +++ b/apps/druid/core/services/runtime_materialization.go @@ -39,27 +39,36 @@ func (s *RuntimeSupervisor) runPullWorker(ctx context.Context, runtimeService po CallbackURL: callbackURL, RegistryCredentials: registryCredentials, } - if err := runtimeService.SpawnPullWorker(waitCtx, action); err != nil { + workerDone, err := runtimeService.SpawnPullWorker(waitCtx, action) + if err != nil { s.workerCallbacks.Cancel(runtimeID) return nil, err } - select { - case result, ok := <-resultCh: - if !ok { - return nil, fmt.Errorf("worker callback closed before result") - } - if result.Error != "" { - return nil, errors.New(result.Error) + for { + select { + case result, ok := <-resultCh: + if !ok { + return nil, fmt.Errorf("worker callback closed before result") + } + if result.Error != "" { + return nil, errors.New(result.Error) + } + return &ports.RuntimeMaterialization{ + Artifact: artifact, + ArtifactDigest: result.ArtifactDigest, + Root: root, + ScrollYAML: []byte(result.ScrollYAML), + }, nil + case workerErr := <-workerDone: + workerDone = nil + if workerErr != nil { + s.workerCallbacks.Cancel(runtimeID) + return nil, workerErr + } + case <-waitCtx.Done(): + s.workerCallbacks.Cancel(runtimeID) + return nil, fmt.Errorf("worker action for runtime %s timed out: %w", runtimeID, waitCtx.Err()) } - return &ports.RuntimeMaterialization{ - Artifact: artifact, - ArtifactDigest: result.ArtifactDigest, - Root: root, - ScrollYAML: []byte(result.ScrollYAML), - }, nil - case <-waitCtx.Done(): - s.workerCallbacks.Cancel(runtimeID) - return nil, fmt.Errorf("worker action for runtime %s timed out: %w", runtimeID, waitCtx.Err()) } } diff --git a/apps/druid/core/services/runtime_supervisor_test.go b/apps/druid/core/services/runtime_supervisor_test.go index fead6e74..46d01199 100644 --- a/apps/druid/core/services/runtime_supervisor_test.go +++ b/apps/druid/core/services/runtime_supervisor_test.go @@ -1,8 +1,10 @@ package services import ( + "bytes" "context" "errors" + "io" "os" "path/filepath" "strings" @@ -16,6 +18,10 @@ import ( "github.com/highcard-dev/daemon/internal/runtime/docker" ) +type testConsoleSession struct{ *bytes.Buffer } + +func (*testConsoleSession) Close() error { return nil } + func TestRuntimeSessionUsesCachedScrollYAML(t *testing.T) { root := t.TempDir() runtimeScroll := &domain.RuntimeScroll{ @@ -46,6 +52,65 @@ commands: } } +func TestRuntimeSupervisorDerivesAndOpensPersistedConsoles(t *testing.T) { + exitCode := 0 + root := t.TempDir() + runtimeScroll := &domain.RuntimeScroll{ + ID: "console-runtime", + Root: root, + ScrollName: "console-runtime", + ScrollYAML: `name: console-runtime +desc: Console runtime +version: 0.1.0 +app_version: "1.0" +commands: + start: + procedures: + - id: server + image: alpine:3.20 + tty: true + - id: stop-server + type: signal + target: server + signal: SIGTERM +`, + Procedures: domain.ProcedureStatusMap{"start": { + "server": {Status: domain.ScrollLockStatusDone, ExitCode: &exitCode}, + "stop-server": {Status: domain.ScrollLockStatusDone, ExitCode: &exitCode}, + }}, + } + store := newTestStateStore(t) + if err := store.CreateScroll(runtimeScroll); err != nil { + t.Fatal(err) + } + opened := &testConsoleSession{Buffer: bytes.NewBufferString("history\n")} + backend := &fakeWorkerBackend{console: opened} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + session, err := NewRuntimeSession(store, runtimeScroll, backend) + if err != nil { + t.Fatal(err) + } + supervisor.sessions[runtimeScroll.ID] = session + + consoles, err := supervisor.Consoles(runtimeScroll.ID) + if err != nil { + t.Fatal(err) + } + if len(consoles) != 1 || consoles["server"].Type != domain.ConsoleTypeTTY || consoles["server"].Exit == nil || *consoles["server"].Exit != 0 { + t.Fatalf("consoles = %#v", consoles) + } + got, err := supervisor.OpenConsole(context.Background(), runtimeScroll.ID, "server") + if err != nil { + t.Fatal(err) + } + if got != opened || backend.consoleRoot != root || backend.consoleProcedure != "server" { + t.Fatalf("open console = (%T, %q, %q)", got, backend.consoleRoot, backend.consoleProcedure) + } + if _, err := supervisor.OpenConsole(context.Background(), runtimeScroll.ID, "stop-server"); err == nil { + t.Fatal("signal procedure unexpectedly exposed a console") + } +} + func TestRuntimeSupervisorRoutesProcedureStatusUpdatesByRuntime(t *testing.T) { store := newTestStateStore(t) backend := &fakeWorkerBackend{} @@ -715,6 +780,36 @@ func TestRuntimeSupervisorCreateWorkerFailureLeavesGeneratedPlaceholder(t *testi } } +func TestRuntimeSupervisorCreateReturnsInfrastructureFailureBeforeCallback(t *testing.T) { + store := newTestStateStore(t) + callbacks := NewWorkerCallbackManager() + done := make(chan error, 1) + done <- errors.New("worker pod is unschedulable") + backend := &fakeWorkerBackend{workerDone: done} + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") + + if _, err := supervisor.Create("registry.local/missing:1.0", "broken-worker", nil); err == nil || !strings.Contains(err.Error(), "unschedulable") { + t.Fatalf("Create error = %v, want infrastructure failure", err) + } +} + +func TestRuntimeSupervisorCreateUsesCallbackWithoutWaitingForWorkerExit(t *testing.T) { + store := newTestStateStore(t) + callbacks := NewWorkerCallbackManager() + backend := &fakeWorkerBackend{ + callbacks: callbacks, + workerDone: make(chan error), + scrollYAML: cachedScrollYAML("start"), + } + supervisor := newRuntimeSupervisorForTest(t, store, coreservices.NewRuntimeScrollManager(store), backend) + supervisor.SetWorkerCallbacks(callbacks, "http://druid-cli:8083") + + if _, err := supervisor.Create("registry.local/lab:1.0", "callback-first", nil); err != nil { + t.Fatal(err) + } +} + func TestRuntimeSupervisorCreateRequiresWorkerCallbackConfig(t *testing.T) { store := newTestStateStore(t) supervisor := newRuntimeSupervisorForTest(t, @@ -1285,12 +1380,16 @@ type fakeWorkerBackend struct { scrollYAML string digest string workerErr error + workerDone <-chan error action ports.RuntimeWorkerAction stopRoot string deleteRoot string spawnCount int runCommand func(ports.RuntimeCommand) (*int, error) stopRuntime func(string) error + console io.ReadWriteCloser + consoleRoot string + consoleProcedure string } func newRuntimeSupervisorForTest(t *testing.T, store ports.RuntimeScrollStore, manager *coreservices.RuntimeScrollManager, factory ports.RuntimeBackendFactory) *RuntimeSupervisor { @@ -1348,8 +1447,10 @@ func (f *fakeWorkerBackend) RoutingTargets(root string, commands map[string]*dom return nil, nil } -func (f *fakeWorkerBackend) Attach(commandName string, data string) error { - return nil +func (f *fakeWorkerBackend) OpenConsole(_ context.Context, root string, procedure string) (io.ReadWriteCloser, error) { + f.consoleRoot = root + f.consoleProcedure = procedure + return f.console, nil } func (f *fakeWorkerBackend) Signal(commandName string, target string, signal string, root string) error { @@ -1377,19 +1478,26 @@ func (f *fakeWorkerBackend) BackupRuntime(ctx context.Context, root string, arti return nil } -func (f *fakeWorkerBackend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) error { +func (f *fakeWorkerBackend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { f.action = action f.spawnCount++ if f.workerErr != nil { - return f.workerErr + return nil, f.workerErr + } + done := f.workerDone + if done == nil { + completed := make(chan error, 1) + completed <- nil + done = completed } if f.callbacks == nil { - return nil + return done, nil } - return f.callbacks.Complete(action.RuntimeID, ports.RuntimeWorkerResult{ + err := f.callbacks.Complete(action.RuntimeID, ports.RuntimeWorkerResult{ ScrollYAML: f.scrollYAML, ArtifactDigest: f.digest, }) + return done, err } func cachedScrollYAML(serve string) string { diff --git a/examples/container-lab/scroll.yaml b/examples/container-lab/scroll.yaml index dbf6a8d9..e7f1a9ce 100644 --- a/examples/container-lab/scroll.yaml +++ b/examples/container-lab/scroll.yaml @@ -50,9 +50,28 @@ commands: date -u '+seeded at %Y-%m-%dT%H:%M:%SZ' > /work/redis/seed.txt; printf 'cache seed refreshed\n' > /work/reports/seed-cache.txt - start: + start-cache: needs: [prepare, seed-cache] run: persistent + procedures: + - id: cache + image: redis:7.4-alpine + expectedPorts: + - name: redis + keepAliveTraffic: 1b/5m + mounts: + - path: /data + sub_path: redis + command: + - redis-server + - --appendonly + - "yes" + - --dir + - /data + + start: + needs: [start-cache] + run: persistent procedures: - id: web image: python:3.12-alpine @@ -71,21 +90,6 @@ commands: - --directory - /site - - id: cache - image: redis:7.4-alpine - expectedPorts: - - name: redis - keepAliveTraffic: 1b/5m - mounts: - - path: /data - sub_path: redis - command: - - redis-server - - --appendonly - - "yes" - - --dir - - /data - verify: needs: [start] run: always diff --git a/internal/api/generated.go b/internal/api/generated.go index 0b8baa1a..914574b4 100644 --- a/internal/api/generated.go +++ b/internal/api/generated.go @@ -227,9 +227,6 @@ type RuntimeUIPackage struct { // RuntimeUIPackages defines model for RuntimeUIPackages. type RuntimeUIPackages map[string]RuntimeUIPackage -// ScrollLogMap defines model for ScrollLogMap. -type ScrollLogMap map[string][]string - // UpdateScrollRequest defines model for UpdateScrollRequest. type UpdateScrollRequest struct { // Artifact Optional target artifact. If omitted, the daemon refreshes the runtime's current artifact. @@ -376,9 +373,6 @@ type ClientInterface interface { // GetScrollConsoles request GetScrollConsoles(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetScrollLogs request - GetScrollLogs(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetScrollPorts request GetScrollPorts(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -574,18 +568,6 @@ func (c *Client) GetScrollConsoles(ctx context.Context, id string, reqEditors .. return c.Client.Do(req) } -func (c *Client) GetScrollLogs(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetScrollLogsRequest(c.Server, id) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - func (c *Client) GetScrollPorts(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetScrollPortsRequest(c.Server, id) if err != nil { @@ -1134,40 +1116,6 @@ func NewGetScrollConsolesRequest(server string, id string) (*http.Request, error return req, nil } -// NewGetScrollLogsRequest generates requests for GetScrollLogs -func NewGetScrollLogsRequest(server string, id string) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/api/v1/scrolls/%s/logs", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - // NewGetScrollPortsRequest generates requests for GetScrollPorts func NewGetScrollPortsRequest(server string, id string) (*http.Request, error) { var err error @@ -1646,9 +1594,6 @@ type ClientWithResponsesInterface interface { // GetScrollConsolesWithResponse request GetScrollConsolesWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollConsolesResponse, error) - // GetScrollLogsWithResponse request - GetScrollLogsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollLogsResponse, error) - // GetScrollPortsWithResponse request GetScrollPortsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollPortsResponse, error) @@ -1909,28 +1854,6 @@ func (r GetScrollConsolesResponse) StatusCode() int { return 0 } -type GetScrollLogsResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *ScrollLogMap -} - -// Status returns HTTPResponse.Status -func (r GetScrollLogsResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetScrollLogsResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - type GetScrollPortsResponse struct { Body []byte HTTPResponse *http.Response @@ -2265,15 +2188,6 @@ func (c *ClientWithResponses) GetScrollConsolesWithResponse(ctx context.Context, return ParseGetScrollConsolesResponse(rsp) } -// GetScrollLogsWithResponse request returning *GetScrollLogsResponse -func (c *ClientWithResponses) GetScrollLogsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollLogsResponse, error) { - rsp, err := c.GetScrollLogs(ctx, id, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetScrollLogsResponse(rsp) -} - // GetScrollPortsWithResponse request returning *GetScrollPortsResponse func (c *ClientWithResponses) GetScrollPortsWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetScrollPortsResponse, error) { rsp, err := c.GetScrollPorts(ctx, id, reqEditors...) @@ -2663,32 +2577,6 @@ func ParseGetScrollConsolesResponse(rsp *http.Response) (*GetScrollConsolesRespo return response, nil } -// ParseGetScrollLogsResponse parses an HTTP response from a GetScrollLogsWithResponse call -func ParseGetScrollLogsResponse(rsp *http.Response) (*GetScrollLogsResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetScrollLogsResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ScrollLogMap - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - // ParseGetScrollPortsResponse parses an HTTP response from a GetScrollPortsWithResponse call func ParseGetScrollPortsResponse(rsp *http.Response) (*GetScrollPortsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -2981,9 +2869,6 @@ type ServerInterface interface { // Get scroll-scoped consoles // (GET /api/v1/scrolls/{id}/consoles) GetScrollConsoles(c *fiber.Ctx, id string) error - // Get scroll-scoped logs - // (GET /api/v1/scrolls/{id}/logs) - GetScrollLogs(c *fiber.Ctx, id string) error // Get runtime scroll port status // (GET /api/v1/scrolls/{id}/ports) GetScrollPorts(c *fiber.Ctx, id string) error @@ -3167,22 +3052,6 @@ func (siw *ServerInterfaceWrapper) GetScrollConsoles(c *fiber.Ctx) error { return siw.Handler.GetScrollConsoles(c, id) } -// GetScrollLogs operation middleware -func (siw *ServerInterfaceWrapper) GetScrollLogs(c *fiber.Ctx) error { - - var err error - - // ------------- Path parameter "id" ------------- - var id string - - err = runtime.BindStyledParameterWithOptions("simple", "id", c.Params("id"), &id, runtime.BindStyledParameterOptions{Explode: false, Required: true}) - if err != nil { - return fiber.NewError(fiber.StatusBadRequest, fmt.Errorf("Invalid format for parameter id: %w", err).Error()) - } - - return siw.Handler.GetScrollLogs(c, id) -} - // GetScrollPorts operation middleware func (siw *ServerInterfaceWrapper) GetScrollPorts(c *fiber.Ctx) error { @@ -3392,8 +3261,6 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option router.Get(options.BaseURL+"/api/v1/scrolls/:id/consoles", wrapper.GetScrollConsoles) - router.Get(options.BaseURL+"/api/v1/scrolls/:id/logs", wrapper.GetScrollLogs) - router.Get(options.BaseURL+"/api/v1/scrolls/:id/ports", wrapper.GetScrollPorts) router.Get(options.BaseURL+"/api/v1/scrolls/:id/queue", wrapper.GetScrollQueue) @@ -3419,55 +3286,54 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xbS3MbNxL+KyjsVu1lRMqbOAftSZE3WSVOWSvZ5UPiYoFAk0Q4A8AARhJXxf++hccM", - "54Hhy1IspXKxRQJoNL5+otF8wFQWSgoQ1uCzB2zoAgri/zxXKl9dy9JyMb+GzyUY675WWirQloOfRIzh", - "c1FUy7mFwv/xdw0zfIb/Nt6QH0fa4+tSWF6AIw3n9Xq8zrBdKcBnmGhNVni9zrCGzyXXwPDZr62tPtVz", - "5fR3oH7xhQZi4YZqmefD/GrLZ4T6EQaGaq4slwKf4XcXl6gaRRpmoEFQQFKjXFKSI+MJI0XsAmcY7kmh", - "8sBsWGNGTJecjebzsQVj/T9n7h9c82qs5mLueOWsz8AbUBooscAQyTkxaCY1EqSAEXrn5zgmLJnmgHRA", - "EHE2DhMuZ0gW3FpgGbILQIxAIQWagwBNLBhEBOJs1GL8dzk1Kd4cxQQ8j8PCv8IYNyonK386ZCzPc0Rl", - "AQbNtCwi0qMVKfL9OTaK0ATbP5dT0ALc/vUsD2zFvwYjS03BjNDlXEgNDE1XSEhx0lg6JXQJgplRand5", - "J0BPUhKNio78DMQZKg0wvzstjZUF6JMZoVzMkXa2gEhpF1Lz/xG3PrmXhjk3Vq8mVAMDYTnJD7C7uPii", - "Xrvb5ipzSRncG8jBAgsW1ze1gEjvCMYSW/oJG8GyQKl/4g473E2JBFIc/VuYUh/iAnrcVYMTxudx9YDx", - "DtrNX+r5PNTzP0Byu7gGo6Qw0NeDQrKERC5KrUFYtPCrUVA25Oc2XZFcps6vtJxrMKZP9iqOIAWagrBk", - "HuScS8Icwo4xj6tzcDOpC2LxGZ7lkrj4UZB7XpQFPnt1eprhgovw6bRmQZTFFHQ0L20njNjE2T4uQDR9", - "s5/rza7e0S08cVqBMyzKPHe+Hp9ZXcIu2/QQpeTwVtLlTW30bRnAPbcTGgUxsB8XFubhcDkxdhJEMqEL", - "IuZ+Xc08F/a7b3FqYcPpCIfcr1iXQrhjZJhJ4WWrtdQ4w3eEu4yncZSBA0eaSa5SOFxJnfBGLQkd4lVU", - "JFfrxnevX3/zuqEdr1JAKC2tpDJvQrGwVuHM/+fDK3WfSqZ2Q+CZi6w0aCdPryUF5ryzB+oXorwvZoyH", - "vOKq7aMHvt/mPxp6tk4w0OeonObcLD5cXhG6JHMYDBg+5duSEPlwc6KltCcacmL5LaDRHTGFTxZH6A3M", - "SJlbg6xESvNbYmHMuLFjolSYJzVSjhva/n6UDIi9gyQcZ+8MCzkQzBQx5k7qdEgrDegBBexogqffWNAg", - "nNKGGHrOo/9+V3m/44L2UNhhAXh8NiO5gezxwpDb0jvPBjtTKXMg4rAYFXFwrmHIRU5lKVhqn8yDPuEq", - "iYkfq3xE3w8sAdR5zm/hvSazGadJGt6xEWr5LberCbEtZ9uMFId7raRnCg4iva7ht/ryv59MVzbAtU8w", - "8BlVkpLtodGAOw4etFe1Ri6307zjgsm7NE+HnG7AQdfY9p11FjVsc/gaoS0a2728JyK7da4g36afhwe8", - "yfDoNgUJznWLOZQ6T/u4befnYv6e6DkkTr/fXeAQ69h1+GNtx0AO1Eq9Ler2VbKLigF9yylM9gsWA1o5", - "GUgnOuS3aOXQVXRr9KC+bsQO8m8DV0DvMEMimRpuXsWGZbgzQiVSqRCRQN8C81q+/63LZ6V+OWHvRL7q", - "JN+bgCflQPANlvDo1b8Mh8RqWOv7SX0UJc4a6b2xUin/XZXhV9WGTwnBlnyiQjq470Hq/NGnnaViBypT", - "qsRR62vEvY1Ftrl6NHS3tfcWG6n5HU50+6gcfKotLrV5Wjcpw7GmeiD/R18UekCkPFpwJW/lfMdNpdb5", - "ISdZq3Nviw/+vMcXrKsLiPVBqC5eD5ZiNcw0mAUY/2WsMP3DIBpLHjWBr1Xa6SDk4wotNberG0co5sNA", - "NOjzMqhq+PRDpZE/fXzvDbyJ008f3yMrlyBCdZl7DuwKKS1vOQPtrcuRd5mZJ7c5v78aO87c+mrPNvmb", - "hdT2xKXSDH0uQa+qzaRGH2F6I+kSLKJSCKBVgYe7hX4yrhKesMVmZ6L4z+BgcdFGzKTbmEphgyqsu4d8", - "o0vO0MXbS5STUtCFr7czVBDhLAX5lVyAPvG1Qla9ZhClck5D4SlDOV/Cb2Lui/IulmiTIUYsmRIDJvME", - "72BajY1+8+xy6wtiNQM4w240sHU6ejU69aFPgSCK4zP8jf8qGL0X6JgoPr59NQ4VN/dNTKnaJ/wRbKXI", - "rdoc9sTD9fGShYmh9Ofl5QOjrwD6zf55elohGdPWBgTj302owgS93aXVnQKjF1Wb5zDDW//r02/+wI1v", - "QsKESkFuCQ9VNW9PZVEQvYpwdnG0ZG78ZT5IIsMBb/zJLa3EFDTHNOTUhv8tN/YmzvlC8A/JJ2Lq13cr", - "PWyqsnd1kDYujv26+m7qc1TQxJEmNi5hNQkgmu+ROIQ9MPZ7yVaPpgipJ891O8a6bG7dk8OrR2OhA/8u", - "uFGVorVRDwfp4L4d9r5KjsE///gYmpRI83noiSSSeoHaSyKnX00iAbWuRMJBOhJBcM+NDaFFxvQjX4V3", - "BHOwuB44Wwc/7/LxvrjC+2ItLkU0KcCCdls8hBgaM8cYQn3u3AY6a4DWTUU/PaEQ2m+ju4VQ3UnWGf72", - "9Nvht7o4XUiLZr5s05Za2PYgO8rSbvxHsC8T+QPV/0sRd3H0C92Ws4Oxy8tKNey7vvfjTyySx/eHuwr9", - "z803BpiRWxoNsu0V74GWDQOLUjtK4lQWBRHMjB/iX+th6V+XIvB8EaY+hQZkSSK03vAgSp2HZ8KtvxD5", - "i2cQPTAUaSMra8DRFGZS+74HJQXjYj4auC+ZlaC4yUX3taf3LvNVvU647LOur/hC73Ndim6I3gjsKJ0U", - "Mz4fzO3roHAR5j3D0NCtIfQEcUW02VyA44H7Pl2lph2LqZE5mL1QDTOfIa7p+lerVjyMeXUwtIRV6GDa", - "1P770McHdUOl8k6iBuUI8HM53wP4t27WC8tzWgXKBObuTMfgnQcsKqzjxwryYaTrZ4ftUF/JcFV4dlgf", - "UmxovNcfXHBADqiq5vLoyWeL+lEW87mEEnbL8b9+2guzmdTLWV9e/mgeQ9iC9+fGrA3QnyMsu+1Fg7Fy", - "W7XiOkz4K+V/6vtgwHnvnL8S3FHW1XgtTUvd/+oi1o/i3Jcj+tRPRp6buIcS8ZbMr0Abbmxsy5X6JPz4", - "BFjs00O6lk1fCXz/wk4VGIdnuz1CZqvX5MXHznbnzD7hMyxAFV6JBCb8JCW2wVeyqRccIaO6uS9tpDdu", - "+E9aHrsJTeHb7cNPepS6l7FSbQNaqj8tzr4vZRfOUnUzvDupl7kkzKC7Bc8BqdD54zSeEUuOE0PJx82+", - "l+0OqdGC8TKl0uzZSVQIQmM4MPThElWouEuUvyGlagWJBejD9VvzxbIYP/g91+O4xbClRKY7Avrj6oQB", - "m210qiat2AKPq7bM1M8Mnig/GWr5X8ck5Zm8yN1xu0CxG6qpUgVY4k28k6yEUyEiEMk1ELY6mZY8tyj0", - "ZHy4RB/Pb36pqBypk6r6SVFa/Zq9TC8oYU21YH1tZXiaSnGg2o0lM57HTqHuRTalG7HRt5LqUAfS+dUl", - "jv1/eIyd6CLRXutUYCI0KRW+BU1sXgXA37vczIaTiVA89ItdyFgNpHBh0K3WYDWHW5JvVvtSVn9tfFKJ", - "F/oNM5uF4VKfeNZItXeh8KKxBN/pFSncwdT4mQkqV1JbxEXoquRS1OIoGwRUaNN9SDYYIboAujTJhbGX", - "p7/0lzK3/CTqQaUWqdNXmtAn8Sa0Y+V8BnRF8/TyqD791T+45OWOWLqoZMbgFnKpvCbEX1xW+LlpCRrn", - "QkgbUHOqjAilYBqnJ/W4wetP6/8HAAD//xVi0p7uQAAA", + "H4sIAAAAAAAC/+xbX3MbNw7/KhzezdzLWnKuTR98T65z7blNJz47mTy0GQ1FQhKrXZIhuZZ1Hn33G/7Z", + "1f7hSpZiN3anL4klkiDwAwiAIHSPqSyUFCCswWf32NAFFMT/ea5Uvr6WpeVifg2fSzDWfa20VKAtBz+J", + "GMPnoqiWcwuF/+PvGmb4DP9tvCU/jrTH16WwvABHGs7r9XiTYbtWgM8w0Zqs8WaTYQ2fS66B4bNfW1t9", + "qufK6e9A/eILDcTCDdUyz4f51ZbPCPUjDAzVXFkuBT7D7y4uUTWKNMxAg6CApEa5pCRHxhNGitgFzjDc", + "kULlgdmwxoyYLjkbzedjC8b6f87cP7jm1VjNxdzxylmfgTegNFBigSGSc2LQTGokSAEj9M7PcUxYMs0B", + "6YAg4mwcJlzOkCy4tcAyZBeAGIFCCjQHAZpYMIgIxNmoxfjvcmpSvDmKCXgeh4V/hTFuVE7WXjpkLM9z", + "RGUBBs20LCLSozUp8odzbBShCbZ/LqegBbj961ke2Ip/DUaWmoIZocu5kBoYmq6RkOKksXRK6BIEM6PU", + "7nIlQE9SGo2GjvwMxBkqDTC/Oy2NlQXokxmhXMyRdmcBkdIupOb/I259ci8Nc26sXk+oBgbCcpIfcO7i", + "4ot67f4zVx2X1IF7AzlYYOHE9Y9aQKQngrHEln7CVrEsUOpL3GGHuymRQIqjfwtT6kNcQI+7anDC+Dyu", + "Hji8g+fmL/N8Hub5HyC5XVyDUVIY6NtBIVlCIxel1iAsWvjVKBgb8nObrkguU/IrLecajOmTvYojSIGm", + "ICyZBz3nkjCHsGPM4+oc3Ezqglh8hme5JC5+FOSOF2WBz16dnma44CJ8Oq1ZEGUxBR2Pl7YTRmxCto8L", + "EE3f7Of6Y1fv6BaeOKvAGRZlnjtfj8+sLmHf2fQQpfTwVtLlTX3o2zqAO24nNCpiYD8uLMyDcDkxdhJU", + "MqELIuZ+Xc08F/a7b3FqYcPpCIfcr1iXQjgxMsyk8LrVWmqc4RXhLuNpiDIgcKSZ5CqFw5XUCW/U0tAh", + "XkVFcrVtfPf69TevG9bxKgWE0tJKKvMmFAtrFc78fz68UvepZGo/BJ65yEqDdlJ6LSkw5509UL8Q5X0x", + "YzzkFVdtHz3w/S7/0bCzTYKBPkflNOdm8eHyitAlmcNgwPAp346EyIebEy2lPdGQE8tvAY1WxBQ+WRyh", + "NzAjZW4NshIpzW+JhTHjxo6JUmGe1Eg5bmj7+1EyIPYESTjOngwLORDMFDFmJXU6pJUG9IABdizB028s", + "aBBOWUMMPefRf7+rvN9xQXso7LAAPD6bkdxA9nhhyG3pnWeDnamUORBxWIyKODjXMOQip7IULLVP5kGf", + "cJXExI9VPqLvB5YA6jznt/Bek9mM0yQN79gItfyW2/WE2JazbUaKw71W0jMFB5Fe1/Bbff3fTaZrG+B6", + "SDDwGVWSku2h0YA7Dh60V7VGLnfTXHHB5CrN0yHSDTjoGtu+s86ihW2FrxHaYbHdy3sislvnCvJd9nl4", + "wJsMj+4ykOBcdxyHUudpH7dLfi7m74meQ0L6h90FDjkd+4Q/9uwYyIFaqXdF3b5JdlExoG85hcnDgsWA", + "VU4G0okO+R1WOXQV3Rk9qK8bsYP828AV0DvMkEimhptXsWEd7o1QiVQqRCTQt8C8lT/81uWzUr+csHci", + "X3eS723Ak3Ig+IaT8OjVvwyHxGrY6vtJfVQlzhrpvbFSKf9dleFX1YZPCcWWfKJCOvhQQer80aedpWIH", + "GlOqxFHba8S9jUW2vXo0bLe1944zUvM7nOj2UTlYqh0utSmtm5ThWFM9kP+jLwo9IFIe7YNn5vhqcnU7", + "sD5C1JXlwTqphpkGswDjv4zln38YRGM9oibwteouHYS806el5nZ94wjFZBWIBn1eBjsKn36ozOWnj+/9", + "6Wvi9NPH98jKJYhQ+uWeA7tGSstbzkB703fkXdrkyW3l9/dWx5lbX+3ZJn+zkNqeuDyXoc8l6HW1mdTo", + "I0xvJF2CRVQKAbSqvnC30E/GVTYSttjuTBT/GRwsLhSImXQbUylsMIVNV8g3uuQMXby9RDkpBV34YjhD", + "BRHOjJFfyQXoE1/IY9VTA1Eq5zRUhTKU8yX8Jua+Yu4cvTYZYsSSKTFgMk9wBdNqbPSbZ5dbX62qGcAZ", + "dqOBrdPRq9Gpj0sKBFEcn+Fv/FfhRHqFjoni49tX41AOc9/EfKct4Y9gK0NuFc6wJx7udpcsTAx1Oa8v", + "H7V8ec5v9s/T0wrJmFM2IBj/bkKJJNjtPqvuVP+8qto8hxk+0rw+/eYP3PgmZDOoFOSW8FDy8uepLAqi", + "1xHOLo6WzI2/aQdNZDjgjT+5pZWaguWYhp7a8L/lxt7EOV8I/iHBPuZlfbfSw6aqSVeCtHFx7NelcVPL", + "UUETR5rYuGzSJIBoPhbiEJPA2O8lWz+aIaTeIzftAOhSrU1PD68ejYUO/PvgRlX+1EY9CNLBfTfsfZMc", + "g3+b8TE0qZHm280TaST1PPQgjZx+NY0E1LoaCYJ0NILgjhsbQouM6Ue+DkV+c7C67jnbBD/vkuW+usLj", + "X60uRTQpwIJ2W9yHGBrTuhhCfWLbBjprgNbNEz89oRLaD5f7lVBdGDYZ/vb02+GHtDhdSItmvqbS1lrY", + "9qBzlKXd+I9gXybyB5r/lyLu4ugXui13DsYuLyvVsO/63o8/sUoe3x/uq8I/N98YYEZuaTyQba94B7Rs", + "HLCotaM0TmVREMHM+D7+tRnW/nUpAs8XYepTWECWJELrDQ+i1HkVJtz6C5G/eAbVA0ORNrKyBhxNYSa1", + "b0pQUjAu5qOB+5JZC4qbXHSfYnqPJl/V64TLPuv6ii/0Ptel6IborcKOskkx4/PB3L4OChdh3jMMDd0a", + "Qk8RV0Sb7QU4Ctz36So17VhMjczBPAjVMPMZ4pqugrUKucOYV4KhJaxDe9G2MN+HPr52GyqVdxI1KEeA", + "X1eqdyN/JUMC++xgP+QK3HjiPfgajBxQVSXg0VOiFvWj9Pi5hBL26/G/ftoLy1hTjy19fXnRPIawA+/P", + "jVlboD9HWCrAh3HWYKzcdYe+DhP+SkSf+pYScH5wJlop7qjT1XhgS2vdN+rHqkac+3JUn/qVwXNT91B6", + "2NL5FWjDjY2dnFKfhN8rAIutXUjXuukbgX/y3msC4/CY9ICQ2WpPePGxs91s8ZDwGRagCq9EGhN+xRA7", + "pyvd1AuO0FHdD5Y+pDdu+E9atLkJfcS7z4ef9CjVGGOl2gW0VH9anH0rwz6cpepmeCupl7kkzKDVgueA", + "VGgWcRbPiCXHqaHk42arxG6H1Hi1f5laabZ5JO6toZcYGPpwiSpU3FXK35NSN9jEAvTh+q35Yl2M7/2e", + "m3HcYvikRKY7CvrjqlcBm110qr6e2DWNq06+VGf6E+UnQ13im5ikPJN3ohW3CxQbaJomVYAl/oh3kpUg", + "FSICkVwDYeuTaclzi0KnwIdL9PH85peKypE2qapfoaTNr9lh84IS1lRj0Nc2hqepXwaq3Vgy43nsX+le", + "ZFO2EXtDK60O9cWcX13i2DKGx9ipLhLtNfQEJkLrTOEbo8S2Vg3+3uVmNpxMhKL3Q6+4JlzKtwS3S8PF", + "PFEwTzUOoVArX4LvIYoUVjA1fmaCypXUFnERmum4FDWkZYOACt2Z98nWFUQXQJcmuTB2ifSX/lLmlp9E", + "XVaqTUlfabNP4k1o9Mn5DOia5unl0QT6q39wCciKWLpw6YfjncEt5FJ5bcYf2lX4uWkJGudCSBtQc+aI", + "CKVgGtKTetzgzafN/wMAAP//k6kvPuU+AAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/core/domain/console.go b/internal/core/domain/console.go index c2d9ff95..673b2261 100644 --- a/internal/core/domain/console.go +++ b/internal/core/domain/console.go @@ -8,22 +8,9 @@ const ( ) type Console struct { - Channel *BroadcastChannel `json:"-" validate:"required"` - - WriteInput func(data string) error `json:"-"` - Type ConsoleType `json:"type" validate:"required"` InputMode string `json:"inputMode" validate:"required"` Exit *int `json:"exit,omitempty"` } //@name Console - -type StreamItem struct { - Data string - Stream string -} - -func (c *Console) MarkExited(exitCode int) { - c.Exit = &exitCode -} diff --git a/internal/core/domain/log.go b/internal/core/domain/log.go deleted file mode 100644 index 43536dc3..00000000 --- a/internal/core/domain/log.go +++ /dev/null @@ -1,10 +0,0 @@ -package domain - -import "container/list" - -type Log struct { - List *list.List - Capacity uint - Req chan chan<- []byte - Write chan<- []byte -} diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index ae5d62b5..9967deea 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -2,6 +2,7 @@ package ports import ( "context" + "io" "time" "github.com/gofiber/fiber/v2" @@ -30,11 +31,6 @@ type ScrollServiceInterface interface { GetCommand(cmd string) (*domain.CommandInstructionSet, error) } -type LogManagerInterface interface { - GetStreams() map[string]*domain.Log - AddLine(stream string, sc []byte) -} - type RuntimeBackendInterface interface { Name() string RootRef(id string, namespace string) string @@ -46,8 +42,8 @@ type RuntimeBackendInterface interface { StopRuntime(root string) error DeleteRuntime(root string, purgeData bool) error BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error - SpawnPullWorker(ctx context.Context, action RuntimeWorkerAction) error - Attach(commandName string, data string) error + SpawnPullWorker(ctx context.Context, action RuntimeWorkerAction) (<-chan error, error) + OpenConsole(ctx context.Context, root string, procedure string) (io.ReadWriteCloser, error) Signal(commandName string, target string, signal string, root string) error } @@ -183,17 +179,6 @@ type RuntimeWorkerResult struct { Error string `json:"error,omitempty"` } -type BroadcastChannelInterface interface { - NewHub() *domain.BroadcastChannel - Run() -} - -type ConsoleManagerInterface interface { - GetConsole(consoleId string) *domain.Console - GetConsoles() map[string]*domain.Console - AddConsoleWithChannel(consoleId string, consoleType domain.ConsoleType, inputMode string, channel chan string) (*domain.Console, chan struct{}) -} - type OciRegistryInterface interface { GetRepo(repoUrl string) (*remote.Repository, error) FetchFile(artifact string, filePath string) ([]byte, error) diff --git a/internal/core/services/console_manager.go b/internal/core/services/console_manager.go deleted file mode 100644 index 8e2168ff..00000000 --- a/internal/core/services/console_manager.go +++ /dev/null @@ -1,60 +0,0 @@ -package services - -import ( - "sync" - - "github.com/highcard-dev/daemon/internal/core/domain" - "github.com/highcard-dev/daemon/internal/core/ports" -) - -type ConsoleManager struct { - consoles map[string]*domain.Console - logManager ports.LogManagerInterface - mu sync.Mutex -} - -func NewConsoleManager(logManager ports.LogManagerInterface) *ConsoleManager { - return &ConsoleManager{ - consoles: make(map[string]*domain.Console), - logManager: logManager, - } -} - -func (cm *ConsoleManager) AddConsoleWithChannel(id string, consoleType domain.ConsoleType, inputMode string, channel chan string) (*domain.Console, chan struct{}) { - cm.mu.Lock() - defer cm.mu.Unlock() - - newChannel := domain.NewHub() - - console := &domain.Console{ - Channel: newChannel, - Type: consoleType, - InputMode: inputMode, - } - - cm.consoles[id] = console - - go newChannel.Run() - - done := make(chan struct{}) - - //broadcast reader into channel (maybe increase chunk size?) - go func() { - for data := range channel { - b := []byte(data) - newChannel.Broadcast(b) - cm.logManager.AddLine(id, b) - } - close(done) - }() - - return console, done -} - -func (cm *ConsoleManager) GetConsoles() map[string]*domain.Console { - return cm.consoles -} - -func (cm *ConsoleManager) GetConsole(id string) *domain.Console { - return cm.consoles[id] -} diff --git a/internal/core/services/log_manager.go b/internal/core/services/log_manager.go deleted file mode 100644 index ff3f9378..00000000 --- a/internal/core/services/log_manager.go +++ /dev/null @@ -1,78 +0,0 @@ -package services - -import ( - "container/list" - - "github.com/highcard-dev/daemon/internal/core/domain" -) - -func NewLog(capacity uint) *domain.Log { - writer := make(chan []byte, 1024) - h := &domain.Log{ - List: list.New(), - Capacity: capacity, - Req: make(chan chan<- []byte), - Write: writer, - } - - go func(write <-chan []byte) { - for { - select { - case res := <-h.Req: - for e := h.List.Front(); e != nil; e = e.Next() { - res <- e.Value.([]byte) - } - close(res) - case in, ok := <-write: - push := func(in []byte, ok bool) bool { - if !ok { - return false - } - if h.List.Len() >= int(h.Capacity) { - h.List.Remove(h.List.Front()) - } - h.List.PushBack(in) - return true - } - push(in, ok) - - //empty entire write queue - drain: - for { - select { - case in, ok = <-write: - if !push(in, ok) { - break drain - } - default: - break drain - } - } - } - } - }(writer) - - return h -} - -type LogManager struct { - Streams map[string]*domain.Log -} - -func NewLogManager() *LogManager { - return &LogManager{ - Streams: make(map[string]*domain.Log), - } -} - -func (hm *LogManager) AddLine(stream string, line []byte) { - - if _, ok := hm.Streams[stream]; !ok { - hm.Streams[stream] = NewLog(100) - } - hm.Streams[stream].Write <- line -} - -func (hm *LogManager) GetStreams() map[string]*domain.Log { - return hm.Streams -} diff --git a/internal/devapi/generated.go b/internal/devapi/generated.go index 46a4299b..7c149155 100644 --- a/internal/devapi/generated.go +++ b/internal/devapi/generated.go @@ -1209,7 +1209,7 @@ type ServerInterface interface { // Return CORS/WebDAV file access options // (OPTIONS /api/v1/files) OptionsFile(c *fiber.Ctx, params OptionsFileParams) error - // Write a file into the runtime root + // Write a file into the runtime root; an optional If-Match SHA-256 ETag serializes Druid-managed writes // (PUT /api/v1/files) PutFile(c *fiber.Ctx, params PutFileParams) error // Stop the in-container build watcher @@ -1438,25 +1438,27 @@ func RegisterHandlersWithOptions(router fiber.Router, si ServerInterface, option // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/8xXTW/bOBP+KwTf96hEyW5OvrVNP3LYNoiBzaEICloc2WwkkhkOnRiB//tiSFm2I+Vj", - "2yzam00OyWeeZ750LyvXemfBUpCTe+kVqhYIMP37YBo4V7Tg3xpChcaTcVZO5EW0ZFo4QOfoAKFRZJYg", - "atOA8IoWhagdCrhTrecVNEtFUHpVXas5HH4Pzh7KQhq+6SYCrmQhrWpBTiSfloVEuIkGQcsJYYRChmoB", - "rWIctPJsFwiNncv1es3GwTsbIGF+q/QF3EQINER9ZpeqMVpgZ7Au5GdHH1y0emjMzgvrSNRpnx/KKNIz", - "l4qqxV9Ow85jHp0HJJOBVK5t1djF7++giqRmDYjaNY27BS1mK2EoCIXz2LIWiR6CNoy4XMjW2LO8eVxs", - "dhWiWvHmLSP7xjyG4dusZhC9YOQELUC0LloCLTCrKljVn0Hg8NrY+TdtECpyuBriuMwmojd5MaaH+u/G", - "ytc958eAFL0sV/1VbvYdqhQMO6LmiBqqCogO+YeNTcMabiJ0QFEgRTGfsbFlcNoEPqFl2kRiQ4avNAPL", - "N18952F37RA+Gxpbu0eTVWhYCqYEULw5P0spmjJWVRWEUIgFqIZzV1ktEpEc/qY2leKLUkySIXZZnmI0", - "WpzCUkwBl4CykEvAkN87Ojw+PGIKnAervJET+WdaKmQflqXyplwelwwgLcwh5RBznZ4703IiPwJxGqaD", - "28L09V7+H6GWE/m/clu+yq1J2Reu9dWD8vDH0VFOTktg04vK+6ZzsXQVAR0EQlAt723LTu2wVSQncmas", - "SoE0lIngjkrfKK5r90+XrJFS00EKTNxJRjnmZe9NuVPp0pGT54/01S5Vs9i27MpEXoDSQuVgqNG1KQF3", - "E4/dVXOmXmbBrtaFXIDSQ8k+gdL/jWYjnMGdCb+EsXcLqK6FqYXqeap3AI3R5RLyMGTsS954ddJOhqR1", - "TwkEimhhGAe8LN59uZiWlzA7ffP3boEQGw/GvPNxJH3P46ukb1LsrdOr3yFz9yeT9UtYT6F6i4YI7A/F", - "6p5Il2gINtlqbNcun8nWddFX3FTZy64XpQbnwoh2p9kgtUT5r4ooj3f7NI552s1S5bDnjhTIZAQoAjnv", - "B2E7JecTCcYeMCZlLKCYRdN0fSz1pw0laWWMErBPM/Le7hPykrD8YS42yr8g4n6ZFgqJtfjZgJ7yRcKh", - "QKicrc08IryCnNv567HRInkyzWa/EasMZ5+hj0DPELI5N85LHuyeouJTtniWhccKZfepx+Po9UiFHbjL", - "k2NIk6MwoRs8V6M9Vm8tFxuQGx+7hezkbdhqn+bW1aMOJ7I/7862Dz0/Pjoe+WiB2dRV10Ai+jkqnRuj", - "p2FBijM+N0sfM6lOVwtl55Am66ybffD6QDa+MLmdO2bERk5kmTpiZ3u/+WbuaFgX/Uou+zsL+dr11fqf", - "AAAA///JD7lP8Q8AAA==", + "H4sIAAAAAAAC/8xYT1PcuBP9Kl36/Y4eDNnsHianJCSBQxIKUsshRaV6rLZHwZacljwwS81332rJ8wds", + "AptlK7kNUkvqfq/7dZsbVbimdZZs8Gp6o1pkbCgQx7/emppOMMzltyZfsGmDcVZN1Wlng2lows6FCVON", + "wSwISlMTtBjmGZSOga6xaWWFzQID5S0Wl1jR3lfv7J7KlJGbvnXES5Upiw2pqZLTKlNM3zrDpNU0cEeZ", + "8sWcGhQ/wrIVOx/Y2EqtVisx9q2znqLPr1Cf0reOfBh6fWwXWBsN3BusMvXBhbeus3poLMGDdQHKuC8P", + "JS/iM+cYivl7p2nnsZZdSxxMcqRwTYNjF7+5pqILOKsJSlfX7oo0zJZgggfkqmuEiwhPoMaPhJypxtjj", + "tHmQrXeRGZeyeSWefREc/fBtYdPDhrDgIMwJGtfZQBo4sQrC6r/xwPGlsdUXbZiK4Hg59OM8mcDG5NE+", + "3eV/N1c+3wp+zJFsQ8vF5io3+0pFTIYdUlNGDVklZsfyw3Z1LRyuM3QAkQ8YunTGdo04p42XE1rFTQ5i", + "KO6jFsfSzRcPRdhfO3RfDI0t3b3FCpoWIJAQw8uT41iisWKxKMj7DOaEtdQuWg0RSEl/U5oC5aKYk8EE", + "CVkdcmc0HNICzogXxCpTC2Kf3tvfO9jbFwhcSxZbo6bqt7iUqU1a5tiafHGQiwNxoaJYQ4J1fO5Yq6l6", + "R0HKMB7cCtPnG/V/plJN1f/yrXzlW5N8I1yrizvy8Gx/PxWnDWTji9i2dR9i7opAYeIDEzayt5Wd0nGD", + "QU3VzFiMiTSkKdB1yNsaRdduvi9ZI1LTu+QFuOfJy7EoN9HkO0oXjzx/+MhG7aKadU0joUzVKaEGTMlQ", + "smtiAe4WnoSLlUCvEmEXq0zNCfWQsiNC/d9wNoIZXRv/UxB7PafiEkwJuMGp3HFoDC4XPfdDxD6mjScH", + "7fkQtP4pYAodWxrmgSzD64+nZ/k5zQ5f/rkrELCOYCy6thvpt2fBtVFxaofaQ5hjgAItMF2xCRTTzGMT", + "c780VZdAiRMEzKh0LNsLY6sXQNeB2GIdHfJLH6iBeAl70C726VYktTAtBgJj4ejTpxN48wkrKJxjbWy8", + "XETsNvwn3ZNoTEyrV04vfwV5uT0+rR6TGrGeBNFA9kcL6uDZPfeutQ2sg9rZihia2GCOy8l7+XEnEc9j", + "evSKZGw/Euwq0gtA2yck1ptr4Ozo5eTZ738k3j2xwdr8RR5iv5o0aLEinRJnLI9X2aYxxQaY9y07zgHO", + "j3Sow2QQJwf1j3qNTMG3iRzDuh858+FoMtJHohEx+ODadlDdsRoFR2Mn4hMaSwyzztR9u49tfA1JXBmD", + "hOz3EXljbwPymML4YSz63HtMzv80LpCDcPEDJXWHPuQAjoFpLZf0BHRux9T7JrAYyVky+4VQFXduI/SO", + "wgOArM+N45Lm3+9BcZQsHkThPqnuv4hlar8c0fhBuDJg+zhgg/H9fL4cHUX01nK+dnIdY7+QgrzyW+7j", + "eL+8N+AI9ofdT4C7kR/sH4x829HszBWXFKBrK0ad5oc2DAWpm8m5Wfzmi1JfzNFWFD9AEm/2zusD2uTC", + "GHbq2R3Xaqry2JN725v1vxZ6GFbZZiXJ/s5CunZ1sfo7AAD//zI33S8YEQAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/internal/runtime/backend.go b/internal/runtime/backend.go index 90439c4c..144672b0 100644 --- a/internal/runtime/backend.go +++ b/internal/runtime/backend.go @@ -33,19 +33,19 @@ func WithDockerConfig(config docker.Config) Option { } } -var newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { - return docker.NewWithConfig(config, consoleManager, observer) +var newDockerBackend = func(config docker.Config, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return docker.NewWithConfig(config, observer) } -var newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { - return runtimekubernetes.New(config, consoleManager, observer) +var newKubernetesBackend = func(config runtimekubernetes.Config, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + return runtimekubernetes.New(config, observer) } var newKubernetesStateStore = func(config runtimekubernetes.Config) (ports.RuntimeScrollStore, error) { return runtimekubernetes.NewConfigMapStateStore(config) } -func NewRuntime(name string, consoleManager ports.ConsoleManagerInterface, stateDir string, opts ...Option) (*Runtime, error) { +func NewRuntime(name string, stateDir string, opts ...Option) (*Runtime, error) { options := Options{} for _, opt := range opts { opt(&options) @@ -59,7 +59,7 @@ func NewRuntime(name string, consoleManager ports.ConsoleManagerInterface, state return &Runtime{ Store: dockerRuntimeStore{RuntimeScrollStore: store, config: options.Docker.WithDefaults()}, create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { - return newDockerBackend(options.Docker, consoleManager, observer) + return newDockerBackend(options.Docker, observer) }, }, nil case "kubernetes": @@ -70,7 +70,7 @@ func NewRuntime(name string, consoleManager ports.ConsoleManagerInterface, state return &Runtime{ Store: store, create: func(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { - return newKubernetesBackend(options.Kubernetes, consoleManager, observer) + return newKubernetesBackend(options.Kubernetes, observer) }, }, nil default: diff --git a/internal/runtime/backend_factory_test.go b/internal/runtime/backend_factory_test.go index 1c3f5cc3..8608a1bd 100644 --- a/internal/runtime/backend_factory_test.go +++ b/internal/runtime/backend_factory_test.go @@ -2,6 +2,7 @@ package runtime import ( "context" + "io" "os" "path/filepath" "strings" @@ -15,7 +16,7 @@ import ( func TestNewRuntimeDockerOwnsStoreSelection(t *testing.T) { previousDocker := newDockerBackend - newDockerBackend = func(config docker.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + newDockerBackend = func(config docker.Config, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { if config.VolumePrefix != "lab" { t.Fatalf("volume prefix = %s, want lab", config.VolumePrefix) } @@ -26,7 +27,7 @@ func TestNewRuntimeDockerOwnsStoreSelection(t *testing.T) { } t.Cleanup(func() { newDockerBackend = previousDocker }) - runtime, err := NewRuntime("docker", nil, t.TempDir(), WithDockerConfig(docker.Config{VolumePrefix: "lab"})) + runtime, err := NewRuntime("docker", t.TempDir(), WithDockerConfig(docker.Config{VolumePrefix: "lab"})) if err != nil { t.Fatal(err) } @@ -45,7 +46,7 @@ func TestNewRuntimeKubernetesOwnsStoreSelection(t *testing.T) { stateDir := t.TempDir() previousBackend := newKubernetesBackend previousStore := newKubernetesStateStore - newKubernetesBackend = func(config runtimekubernetes.Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { + newKubernetesBackend = func(config runtimekubernetes.Config, observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { if config.Namespace != "druid" { t.Fatalf("backend namespace = %s, want druid", config.Namespace) } @@ -65,7 +66,7 @@ func TestNewRuntimeKubernetesOwnsStoreSelection(t *testing.T) { newKubernetesStateStore = previousStore }) - runtime, err := NewRuntime("kubernetes", nil, stateDir, WithKubernetesConfig(runtimekubernetes.Config{Namespace: "druid"})) + runtime, err := NewRuntime("kubernetes", stateDir, WithKubernetesConfig(runtimekubernetes.Config{Namespace: "druid"})) if err != nil { t.Fatal(err) } @@ -84,7 +85,7 @@ func TestNewRuntimeKubernetesOwnsStoreSelection(t *testing.T) { } func TestNewRuntimeUnknownBackendErrorsOnce(t *testing.T) { - _, err := NewRuntime("nope", nil, t.TempDir()) + _, err := NewRuntime("nope", t.TempDir()) if err == nil || !strings.Contains(err.Error(), `unknown runtime backend "nope"`) { t.Fatalf("error = %v", err) } @@ -165,12 +166,14 @@ func (f fakeBackend) BackupRuntime(ctx context.Context, root string, artifact st return nil } -func (f fakeBackend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) error { - return nil +func (f fakeBackend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { + done := make(chan error, 1) + done <- nil + return done, nil } -func (f fakeBackend) Attach(commandName string, data string) error { - return nil +func (f fakeBackend) OpenConsole(context.Context, string, string) (io.ReadWriteCloser, error) { + return nil, nil } func (f fakeBackend) Signal(commandName string, target string, signal string, root string) error { diff --git a/internal/runtime/docker/backend.go b/internal/runtime/docker/backend.go index 525c2cdb..a8731a56 100644 --- a/internal/runtime/docker/backend.go +++ b/internal/runtime/docker/backend.go @@ -2,7 +2,6 @@ package docker import ( "fmt" - "io" "os" "path/filepath" "sync" @@ -13,12 +12,10 @@ import ( type Backend struct { client *client.Client - consoleManager ports.ConsoleManagerInterface procedureStatusObserver ports.ProcedureStatusObserver config Config mu sync.Mutex containers map[string]string - stdin map[string]io.Writer } type Config struct { @@ -93,11 +90,11 @@ func (c Config) ValidateForUIPublishing() error { return nil } -func New(consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { - return NewWithConfig(Config{}, consoleManager, observer) +func New(observer ports.ProcedureStatusObserver) (*Backend, error) { + return NewWithConfig(Config{}, observer) } -func NewWithConfig(config Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { +func NewWithConfig(config Config, observer ports.ProcedureStatusObserver) (*Backend, error) { if observer == nil { return nil, fmt.Errorf("procedure status observer is required") } @@ -119,11 +116,9 @@ func NewWithConfig(config Config, consoleManager ports.ConsoleManagerInterface, } return &Backend{ client: cli, - consoleManager: consoleManager, procedureStatusObserver: observer, config: config, containers: map[string]string{}, - stdin: map[string]io.Writer{}, }, nil } diff --git a/internal/runtime/docker/console.go b/internal/runtime/docker/console.go index d6347604..d7500839 100644 --- a/internal/runtime/docker/console.go +++ b/internal/runtime/docker/console.go @@ -4,32 +4,115 @@ import ( "context" "fmt" "io" + "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/pkg/stdcopy" ) -func (b *Backend) Attach(commandName string, data string) error { - b.mu.Lock() - stdin := b.stdin[commandName] - b.mu.Unlock() - if stdin == nil { - return fmt.Errorf("target container %s not attached", commandName) +type consoleSession struct { + io.ReadCloser + write func([]byte) (int, error) + close func() +} + +func (s *consoleSession) Write(data []byte) (int, error) { + if s.write == nil { + return 0, fmt.Errorf("console input is unavailable") + } + return s.write(data) +} + +func (s *consoleSession) Close() error { + if s.close != nil { + s.close() } - _, err := stdin.Write([]byte(data)) - return err + return s.ReadCloser.Close() +} + +func (b *Backend) OpenConsole(ctx context.Context, root string, procedure string) (io.ReadWriteCloser, error) { + selector := filters.NewArgs( + filters.Arg("label", dockerLabelRole+"="+dockerRoleProcedure), + filters.Arg("label", dockerLabelRootHash+"="+rootHash(root)), + filters.Arg("label", dockerLabelProcedure+"="+procedure), + ) + var items []container.Summary + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + poll := time.NewTicker(250 * time.Millisecond) + defer poll.Stop() + for len(items) == 0 { + var err error + items, err = b.client.ContainerList(ctx, container.ListOptions{All: true, Filters: selector}) + if err != nil { + return nil, err + } + if len(items) > 0 { + break + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-deadline.C: + return nil, fmt.Errorf("no container found for procedure %s", procedure) + case <-poll.C: + } + } + selected := selectConsoleContainer(items) + inspected, err := b.client.ContainerInspect(ctx, selected.ID) + if err != nil { + return nil, err + } + running := inspected.State != nil && inspected.State.Running + logs, err := b.client.ContainerLogs(ctx, selected.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true, Follow: running}) + if err != nil { + return nil, err + } + output := io.ReadCloser(logs) + if inspected.Config != nil && !inspected.Config.Tty { + reader, writer := io.Pipe() + go func() { + _, copyErr := stdcopy.StdCopy(writer, writer, logs) + _ = logs.Close() + _ = writer.CloseWithError(copyErr) + }() + output = reader + } + session := &consoleSession{ReadCloser: output} + if running { + attach, attachErr := b.client.ContainerAttach(ctx, selected.ID, container.AttachOptions{Stream: true, Stdin: true}) + if attachErr != nil { + _ = output.Close() + return nil, attachErr + } + session.write = attach.Conn.Write + session.close = func() { attach.Close() } + } + return session, nil +} + +func selectConsoleContainer(items []container.Summary) container.Summary { + selected := items[0] + for _, candidate := range items[1:] { + candidateRunning := candidate.State == "running" + selectedRunning := selected.State == "running" + if candidateRunning && !selectedRunning || candidateRunning == selectedRunning && candidate.Created > selected.Created { + selected = candidate + } + } + return selected } func (b *Backend) Signal(_ string, target string, signal string, root string) error { if target == "" { return nil } - ctx := context.Background() - containerID := b.containerID(target, root) options := container.StopOptions{} if signal != "" { options.Signal = signal } - return b.client.ContainerStop(ctx, containerID, options) + return b.client.ContainerStop(context.Background(), b.containerID(target, root), options) } func (b *Backend) containerID(commandName string, root string) string { @@ -52,26 +135,3 @@ func (b *Backend) clearContainer(commandName string) { defer b.mu.Unlock() delete(b.containers, commandName) } - -func (b *Backend) setStdin(commandName string, stdin io.Writer) { - b.mu.Lock() - defer b.mu.Unlock() - b.stdin[commandName] = stdin -} - -func (b *Backend) clearStdin(commandName string) { - b.mu.Lock() - defer b.mu.Unlock() - delete(b.stdin, commandName) -} - -type channelWriter struct { - channel chan<- string -} - -func (w channelWriter) Write(p []byte) (int, error) { - if len(p) > 0 { - w.channel <- string(p) - } - return len(p), nil -} diff --git a/internal/runtime/docker/console_test.go b/internal/runtime/docker/console_test.go new file mode 100644 index 00000000..a51ca136 --- /dev/null +++ b/internal/runtime/docker/console_test.go @@ -0,0 +1,26 @@ +package docker + +import ( + "testing" + + "github.com/docker/docker/api/types/container" +) + +func TestSelectConsoleContainerPrefersRunningThenNewest(t *testing.T) { + selected := selectConsoleContainer([]container.Summary{ + {ID: "old-stopped", State: "exited", Created: 1}, + {ID: "running", State: "running", Created: 2}, + {ID: "new-stopped", State: "exited", Created: 3}, + }) + if selected.ID != "running" { + t.Fatalf("selected = %s, want running", selected.ID) + } + + selected = selectConsoleContainer([]container.Summary{ + {ID: "old", State: "exited", Created: 1}, + {ID: "new", State: "exited", Created: 2}, + }) + if selected.ID != "new" { + t.Fatalf("selected = %s, want new", selected.ID) + } +} diff --git a/internal/runtime/docker/lifecycle.go b/internal/runtime/docker/lifecycle.go index 1f01befc..589f0f5e 100644 --- a/internal/runtime/docker/lifecycle.go +++ b/internal/runtime/docker/lifecycle.go @@ -29,9 +29,6 @@ func (b *Backend) StopRuntime(root string) error { for key := range b.containers { delete(b.containers, key) } - for key := range b.stdin { - delete(b.stdin, key) - } b.mu.Unlock() return nil } @@ -61,7 +58,6 @@ func (b *Backend) StopCommand(root string, command string) error { for _, item := range items { if id == item.ID { delete(b.containers, key) - delete(b.stdin, key) } } } diff --git a/internal/runtime/docker/names.go b/internal/runtime/docker/names.go index 735b0a4b..7b8b8a2f 100644 --- a/internal/runtime/docker/names.go +++ b/internal/runtime/docker/names.go @@ -63,13 +63,6 @@ func rootHash(root string) string { return hex.EncodeToString(hash[:])[:10] } -func runtimeConsoleID(scrollID string, procedureName string) string { - if scrollID == "" { - return procedureName - } - return scrollID + "/" + procedureName -} - func sanitizeContainerName(name string) string { re := regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) name = re.ReplaceAllString(name, "-") diff --git a/internal/runtime/docker/procedures.go b/internal/runtime/docker/procedures.go index fb2ecee6..a937e9d1 100644 --- a/internal/runtime/docker/procedures.go +++ b/internal/runtime/docker/procedures.go @@ -4,11 +4,8 @@ import ( "context" "errors" "fmt" - "io" - "sync" "github.com/docker/docker/api/types/container" - "github.com/docker/docker/pkg/stdcopy" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" "github.com/highcard-dev/daemon/internal/utils/logger" @@ -46,14 +43,14 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { return nil, fmt.Errorf("docker runtime procedure %s requires image", procedureName) } b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) - if err := b.startPersistentContainer(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env); err != nil { + if err := b.startPersistentContainer(command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env); err != nil { b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) return nil, err } continue } b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) - exitCode, err := b.runProcedure(runtimeConsoleID(command.ScrollID, procedureName), command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env) + exitCode, err := b.runProcedure(command.Name, procedureName, procedureResourceName(command.Name, idx), procedure, command.Root, command.Ports, command.Routing, env) if err != nil { if exitCode != nil && *exitCode != 0 && procedure.IgnoreFailure { b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, exitCode) @@ -88,17 +85,17 @@ func (b *Backend) observeProcedureStatus(command ports.RuntimeCommand, procedure }) } -func (b *Backend) runProcedure(consoleID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) (*int, error) { +func (b *Backend) runProcedure(commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) (*int, error) { if procedure.IsSignal() { return nil, b.Signal(procedureName, procedure.Target, procedure.Signal, root) } if procedure.Image == "" { return nil, fmt.Errorf("docker runtime procedure %s requires image", procedureName) } - return b.runContainer(consoleID, commandName, procedureName, resourceName, procedure, root, globalPorts, routing, env) + return b.runContainer(commandName, procedureName, resourceName, procedure, root, globalPorts, routing, env) } -func (b *Backend) runContainer(consoleID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) (*int, error) { +func (b *Backend) runContainer(commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) (*int, error) { ctx := context.Background() if procedure.Image == "" { return nil, errors.New("docker image is required") @@ -128,42 +125,6 @@ func (b *Backend) runContainer(consoleID string, commandName string, procedureNa b.clearContainer(procedureName) }() - attach, err := b.client.ContainerAttach(ctx, selected.ID, container.AttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - return nil, err - } - defer attach.Close() - b.setStdin(procedureName, attach.Conn) - defer b.clearStdin(procedureName) - - combined := make(chan string, 20) - consoleType := domain.ConsoleTypeContainer - if procedure.TTY { - consoleType = domain.ConsoleTypeTTY - } - console, doneChan := b.consoleManager.AddConsoleWithChannel(consoleID, consoleType, "stdin", combined) - console.WriteInput = func(data string) error { - return b.Attach(procedureName, data) - } - - var copyWG sync.WaitGroup - copyWG.Add(1) - go func() { - defer copyWG.Done() - defer close(combined) - writer := channelWriter{channel: combined} - if procedure.TTY { - _, _ = io.Copy(writer, attach.Reader) - return - } - _, _ = stdcopy.StdCopy(writer, writer, attach.Reader) - }() - if selected.Start { if err := b.client.ContainerStart(ctx, selected.ID, container.StartOptions{}); err != nil { _ = b.client.ContainerRemove(context.Background(), selected.ID, container.RemoveOptions{Force: true}) @@ -181,12 +142,7 @@ func (b *Backend) runContainer(consoleID string, commandName string, procedureNa case status := <-statusCh: exitCode = int(status.StatusCode) } - _ = attach.CloseWrite() - copyWG.Wait() - console.MarkExited(exitCode) - <-doneChan if exitCode == 0 { - _ = b.client.ContainerRemove(context.Background(), selected.ID, container.RemoveOptions{Force: true}) return &exitCode, nil } return &exitCode, &domain.CommandExecutionError{ @@ -196,7 +152,7 @@ func (b *Backend) runContainer(consoleID string, commandName string, procedureNa } } -func (b *Backend) startPersistentContainer(consoleID string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) error { +func (b *Backend) startPersistentContainer(commandName string, procedureName string, resourceName string, procedure *domain.Procedure, root string, globalPorts []domain.Port, routing []domain.RuntimeRouteAssignment, env map[string]string) error { ctx := context.Background() if procedure.Image == "" { return errors.New("docker image is required") @@ -219,47 +175,11 @@ func (b *Backend) startPersistentContainer(consoleID string, commandName string, if err != nil { return dockerSetupError(err) } - attach, err := b.client.ContainerAttach(ctx, selected.ID, container.AttachOptions{ - Stream: true, - Stdin: true, - Stdout: true, - Stderr: true, - }) - if err != nil { - if selected.Start { - _ = b.client.ContainerRemove(context.Background(), selected.ID, container.RemoveOptions{Force: true}) - } - return err - } b.setContainer(procedureName, selected.ID) - b.setStdin(procedureName, attach.Conn) - - combined := make(chan string, 20) - consoleType := domain.ConsoleTypeContainer - if procedure.TTY { - consoleType = domain.ConsoleTypeTTY - } - console, _ := b.consoleManager.AddConsoleWithChannel(consoleID, consoleType, "stdin", combined) - console.WriteInput = func(data string) error { - return b.Attach(procedureName, data) - } - - go func() { - defer close(combined) - defer attach.Close() - writer := channelWriter{channel: combined} - if procedure.TTY { - _, _ = io.Copy(writer, attach.Reader) - return - } - _, _ = stdcopy.StdCopy(writer, writer, attach.Reader) - }() if selected.Start { if err := b.client.ContainerStart(ctx, selected.ID, container.StartOptions{}); err != nil { - attach.Close() b.clearContainer(procedureName) - b.clearStdin(procedureName) _ = b.client.ContainerRemove(context.Background(), selected.ID, container.RemoveOptions{Force: true}) return dockerSetupError(err) } @@ -271,10 +191,7 @@ func (b *Backend) startPersistentContainer(consoleID string, commandName string, case <-errCh: case status := <-statusCh: exitCode := int(status.StatusCode) - console.MarkExited(exitCode) - if exitCode == 0 { - _ = b.client.ContainerRemove(context.Background(), selected.ID, container.RemoveOptions{Force: true}) - } else { + if exitCode != 0 { logger.Log().Error("Docker persistent procedure exited", zap.String("container", selected.Name), zap.String("container_id", selected.ID), @@ -285,7 +202,6 @@ func (b *Backend) startPersistentContainer(consoleID string, commandName string, } } b.clearContainer(procedureName) - b.clearStdin(procedureName) }() return nil } diff --git a/internal/runtime/docker/workers.go b/internal/runtime/docker/workers.go index 1ddebec2..dbacedf8 100644 --- a/internal/runtime/docker/workers.go +++ b/internal/runtime/docker/workers.go @@ -95,39 +95,39 @@ func (b *Backend) runWorkerRootCommand(ctx context.Context, root string, command return nil } -func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) error { +func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { if b.config.WorkerImage == "" { - return fmt.Errorf("docker worker image is required; set --docker-worker-image or DRUID_DOCKER_WORKER_IMAGE") + return nil, fmt.Errorf("docker worker image is required; set --docker-worker-image or DRUID_DOCKER_WORKER_IMAGE") } root := action.RootRef if root == "" { - return fmt.Errorf("worker root ref is required") + return nil, fmt.Errorf("worker root ref is required") } if action.MountPath == "" { action.MountPath = "/scroll" } if err := b.pullImage(ctx, b.config.WorkerImage); err != nil { - return err + return nil, err } if err := b.prepareWritableRoot(ctx, root); err != nil { - return err + return nil, err } registryConfig, err := json.Marshal(struct { Registries []domain.RegistryCredential `json:"registries"` }{Registries: action.RegistryCredentials}) if err != nil { - return err + return nil, err } rootMount, err := DockerMount(root, action.MountPath, false, "") if err != nil { - return err + return nil, err } artifact := action.Artifact mounts := []mount.Mount{rootMount} if info, statErr := os.Stat(action.Artifact); statErr == nil { abs, err := filepath.Abs(action.Artifact) if err != nil { - return err + return nil, err } if info.IsDir() { mounts = append(mounts, mount.Mount{Type: mount.TypeBind, Source: abs, Target: "/artifact-src", ReadOnly: true}) @@ -165,32 +165,33 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke }, }, hostConfig, nil, nil, name) if err != nil { - return err + return nil, err } - defer b.client.ContainerRemove(context.Background(), created.ID, container.RemoveOptions{Force: true}) if err := b.client.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil { - return err + _ = b.client.ContainerRemove(context.Background(), created.ID, container.RemoveOptions{Force: true}) + return nil, err } statusCh, errCh := b.client.ContainerWait(ctx, created.ID, container.WaitConditionNotRunning) - select { - case err := <-errCh: - if err != nil { - return err - } - case status := <-statusCh: - if status.StatusCode != 0 { - logs, _ := b.client.ContainerLogs(context.Background(), created.ID, container.LogsOptions{ShowStdout: true, ShowStderr: true}) - defer func() { - if logs != nil { - logs.Close() + done := make(chan error, 1) + go func() { + defer b.client.ContainerRemove(context.Background(), created.ID, container.RemoveOptions{Force: true}) + for { + select { + case err, ok := <-errCh: + if ok && err != nil { + done <- err + return } - }() - var message strings.Builder - if logs != nil { - _, _ = io.Copy(&message, logs) + errCh = nil + case status := <-statusCh: + if status.StatusCode != 0 { + done <- fmt.Errorf("worker container exited with %d", status.StatusCode) + return + } + done <- nil + return } - return fmt.Errorf("worker container exited with %d: %s", status.StatusCode, strings.TrimSpace(message.String())) } - } - return nil + }() + return done, nil } diff --git a/internal/runtime/kubernetes/attach_signal.go b/internal/runtime/kubernetes/attach_signal.go index d234ad1d..fb1dffd1 100644 --- a/internal/runtime/kubernetes/attach_signal.go +++ b/internal/runtime/kubernetes/attach_signal.go @@ -3,7 +3,9 @@ package kubernetes import ( "context" "fmt" + "io" "strings" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,43 +17,132 @@ import ( "go.uber.org/zap" ) -func (b *Backend) Attach(commandName string, data string) error { - logger.Log().Debug("Attaching to Kubernetes procedure by command name", zap.String("command", commandName), zap.Int("bytes", len(data))) - pods, err := b.client.CoreV1().Pods(b.config.Namespace).List(context.Background(), metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set{labelProcedure: dnsLabel(commandName)}).String(), - }) - if err != nil { - logger.Log().Error("Failed to list Kubernetes pods for attach", zap.String("namespace", b.config.Namespace), zap.String("command", commandName), zap.Error(err)) - return err - } - for _, pod := range pods.Items { - if pod.Status.Phase == corev1.PodRunning { - logger.Log().Debug("Attaching to Kubernetes pod", zap.String("namespace", b.config.Namespace), zap.String("pod", pod.Name), zap.String("command", commandName), zap.Int("bytes", len(data))) - return b.attachToPod(context.Background(), b.config.Namespace, pod.Name, data) - } +type consoleSession struct { + io.ReadCloser + write func([]byte) (int, error) +} + +func (s *consoleSession) Write(data []byte) (int, error) { + if s.write == nil { + return 0, fmt.Errorf("console input is unavailable") } - logger.Log().Warn("No running Kubernetes pod found for attach", zap.String("namespace", b.config.Namespace), zap.String("command", commandName), zap.Int("pods", len(pods.Items))) - return fmt.Errorf("no running pod found for console %s", commandName) + return s.write(data) } -func (b *Backend) attachToProcedure(root string, procedureName string, data string) error { +func (b *Backend) OpenConsole(ctx context.Context, root string, procedure string) (io.ReadWriteCloser, error) { namespace, pvc, err := parseRef(root) if err != nil { - logger.Log().Error("Cannot attach to Kubernetes procedure for invalid root", zap.String("root", root), zap.String("procedure", procedureName), zap.Error(err)) - return err + return nil, err } selector := baseLabels(pvc) - selector[labelProcedure] = dnsLabel(procedureName) - podName, err := b.waitForPodBySelector(context.Background(), namespace, labels.SelectorFromSet(selector).String()) + labelSelector := labels.SelectorFromSet(selector).String() + pod, containerName, running, err := b.waitForConsoleContainer(ctx, namespace, labelSelector, procedure) if err != nil { - logger.Log().Error("Failed to find Kubernetes procedure pod for attach", zap.String("namespace", namespace), zap.String("procedure", procedureName), zap.Any("selector", selector), zap.Error(err)) - return err + return nil, err + } + if pod == nil || containerName == "" { + return nil, fmt.Errorf("no pod found for procedure %s", procedure) + } + stream, err := b.client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: containerName, + Follow: running, + }).Stream(ctx) + if err != nil { + return nil, err + } + session := &consoleSession{ReadCloser: stream} + if running { + session.write = func(data []byte) (int, error) { + if err := b.attachToPod(ctx, namespace, pod.Name, containerName, string(data)); err != nil { + return 0, err + } + return len(data), nil + } + } + return session, nil +} + +func (b *Backend) waitForConsoleContainer(ctx context.Context, namespace string, selector string, procedure string) (*corev1.Pod, string, bool, error) { + deadline := time.Now().Add(5 * time.Second) + for { + pods, err := b.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, "", false, err + } + pod, containerName, running := selectProcedureConsole(pods.Items, procedure) + if pod != nil { + return pod, containerName, running, nil + } + if time.Now().After(deadline) { + return nil, "", false, fmt.Errorf("no pod found for procedure %s", procedure) + } + select { + case <-ctx.Done(): + return nil, "", false, ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } +} + +func selectProcedureConsole(pods []corev1.Pod, procedure string) (*corev1.Pod, string, bool) { + type candidate struct { + pod *corev1.Pod + container string + running bool + } + candidates := []candidate{} + for idx := range pods { + pod := &pods[idx] + if pod.Labels[labelProcedure] == dnsLabel(procedure) { + candidates = append(candidates, candidate{pod: pod, container: "main", running: containerIsRunning(pod.Status.ContainerStatuses, "main")}) + continue + } + initName := persistentInitContainerName(procedure) + for _, container := range pod.Spec.InitContainers { + if container.Name == initName { + candidates = append(candidates, candidate{pod: pod, container: initName, running: containerIsRunning(pod.Status.InitContainerStatuses, initName)}) + break + } + } + } + if len(candidates) == 0 { + return nil, "", false } - logger.Log().Debug("Attaching to Kubernetes procedure pod", zap.String("namespace", namespace), zap.String("pod", podName), zap.String("procedure", procedureName), zap.Int("bytes", len(data))) - return b.attachToPod(context.Background(), namespace, podName, data) + selected := candidates[0] + for _, candidate := range candidates[1:] { + if candidate.running && !selected.running || candidate.running == selected.running && candidate.pod.CreationTimestamp.After(selected.pod.CreationTimestamp.Time) { + selected = candidate + } + } + return selected.pod, selected.container, selected.running } -func (b *Backend) attachToPod(ctx context.Context, namespace string, podName string, data string) error { +func containerIsRunning(statuses []corev1.ContainerStatus, name string) bool { + for _, status := range statuses { + if status.Name == name { + return status.State.Running != nil + } + } + return false +} + +func selectConsolePod(pods []corev1.Pod) *corev1.Pod { + if len(pods) == 0 { + return nil + } + selected := &pods[0] + for index := 1; index < len(pods); index++ { + candidate := &pods[index] + candidateRunning := candidate.Status.Phase == corev1.PodRunning + selectedRunning := selected.Status.Phase == corev1.PodRunning + if candidateRunning && !selectedRunning || candidateRunning == selectedRunning && candidate.CreationTimestamp.After(selected.CreationTimestamp.Time) { + selected = candidate + } + } + return selected +} + +func (b *Backend) attachToPod(ctx context.Context, namespace string, podName string, containerName string, data string) error { logger.Log().Debug("Opening Kubernetes pod attach stream", zap.String("namespace", namespace), zap.String("pod", podName), zap.Int("bytes", len(data))) req := b.client.CoreV1().RESTClient().Post(). Resource("pods"). @@ -59,7 +150,7 @@ func (b *Backend) attachToPod(ctx context.Context, namespace string, podName str Name(podName). SubResource("attach"). VersionedParams(&corev1.PodAttachOptions{ - Container: "main", + Container: containerName, Stdin: true, Stdout: false, Stderr: false, @@ -99,10 +190,3 @@ func (b *Backend) Signal(_ string, target string, signal string, root string) er return fmt.Errorf("kubernetes signal %s is unsupported without pod exec", signal) } } - -func runtimeConsoleID(scrollID string, procedureName string) string { - if scrollID == "" { - return procedureName - } - return scrollID + "/" + procedureName -} diff --git a/internal/runtime/kubernetes/attach_signal_test.go b/internal/runtime/kubernetes/attach_signal_test.go new file mode 100644 index 00000000..ee69e652 --- /dev/null +++ b/internal/runtime/kubernetes/attach_signal_test.go @@ -0,0 +1,63 @@ +package kubernetes + +import ( + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSelectConsolePodPrefersRunningThenNewest(t *testing.T) { + selected := selectConsolePod([]corev1.Pod{ + consolePod("old-stopped", corev1.PodSucceeded, 1), + consolePod("running", corev1.PodRunning, 2), + consolePod("new-stopped", corev1.PodSucceeded, 3), + }) + if selected == nil || selected.Name != "running" { + t.Fatalf("selected = %#v, want running", selected) + } + + selected = selectConsolePod([]corev1.Pod{ + consolePod("old", corev1.PodSucceeded, 1), + consolePod("new", corev1.PodSucceeded, 2), + }) + if selected == nil || selected.Name != "new" { + t.Fatalf("selected = %#v, want new", selected) + } +} + +func TestSelectProcedureConsoleFindsRunningPersistentInitContainer(t *testing.T) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "minecraft-start-1-0", + Labels: map[string]string{labelProcedure: "start"}, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "init-coldstart"}}, + Containers: []corev1.Container{{Name: "main"}}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + InitContainerStatuses: []corev1.ContainerStatus{{ + Name: "init-coldstart", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } + selected, container, running := selectProcedureConsole([]corev1.Pod{pod}, "coldstart") + if selected == nil || selected.Name != pod.Name || container != "init-coldstart" || !running { + t.Fatalf("selected = %#v, container = %q, running = %v", selected, container, running) + } + selected, container, running = selectProcedureConsole([]corev1.Pod{pod}, "start") + if selected == nil || container != "main" || running { + t.Fatalf("selected = %#v, container = %q, running = %v", selected, container, running) + } +} + +func consolePod(name string, phase corev1.PodPhase, created int64) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, CreationTimestamp: metav1.NewTime(time.Unix(created, 0))}, + Status: corev1.PodStatus{Phase: phase}, + } +} diff --git a/internal/runtime/kubernetes/backend.go b/internal/runtime/kubernetes/backend.go index 8480ce5f..4ff74b63 100644 --- a/internal/runtime/kubernetes/backend.go +++ b/internal/runtime/kubernetes/backend.go @@ -21,11 +21,10 @@ type Backend struct { client k8sclient.Interface restConfig *rest.Config httpClient *http.Client - consoleManager ports.ConsoleManagerInterface procedureStatusObserver ports.ProcedureStatusObserver config Config statsReader nodeStatsReader - jobLogRunner func(context.Context, *batchv1.Job) ([]byte, error) + helperJobRunner func(context.Context, *batchv1.Job) error jobExitMu sync.Mutex jobExits map[string]recentJobExit } @@ -40,7 +39,7 @@ const ( defaultKubernetesClientBurst int = 100 ) -func New(config Config, consoleManager ports.ConsoleManagerInterface, observer ports.ProcedureStatusObserver) (*Backend, error) { +func New(config Config, observer ports.ProcedureStatusObserver) (*Backend, error) { if observer == nil { return nil, fmt.Errorf("procedure status observer is required") } @@ -71,7 +70,6 @@ func New(config Config, consoleManager ports.ConsoleManagerInterface, observer p client: client, restConfig: restConfig, httpClient: httpClient, - consoleManager: consoleManager, procedureStatusObserver: observer, config: config, jobExits: make(map[string]recentJobExit), @@ -140,12 +138,12 @@ func kubeconfigRESTConfig(config Config) (*rest.Config, string, string, error) { return restConfig, namespace, source, nil } -func NewWithClient(config Config, consoleManager ports.ConsoleManagerInterface, client k8sclient.Interface) *Backend { +func NewWithClient(config Config, client k8sclient.Interface) *Backend { config = config.WithDefaults() if config.Namespace == "" { config.Namespace = "default" } - backend := &Backend{client: client, consoleManager: consoleManager, config: config, jobExits: make(map[string]recentJobExit)} + backend := &Backend{client: client, config: config, jobExits: make(map[string]recentJobExit)} backend.statsReader = backend.readNodeStatsSummary return backend } diff --git a/internal/runtime/kubernetes/ports.go b/internal/runtime/kubernetes/ports.go index 63db33cc..fe835b43 100644 --- a/internal/runtime/kubernetes/ports.go +++ b/internal/runtime/kubernetes/ports.go @@ -37,7 +37,12 @@ func (b *Backend) ExpectedPorts(root string, commands map[string]*domain.Command if procedure.Id != nil { procedureName = *procedure.Id } - traffic, trafficErr := b.procedureTrafficForSelector(context.Background(), namespace, procedureSelector(pvc, commandName, procedureName), now) + trafficSelector := procedureSelector(pvc, commandName, procedureName) + if command.Run == domain.RunModePersistent { + trafficSelector = baseLabels(pvc) + trafficSelector[labelCommand] = dnsLabel(commandName) + } + traffic, trafficErr := b.procedureTrafficForSelector(context.Background(), namespace, trafficSelector, now) for _, expectedPort := range procedure.ExpectedPorts { port := portsByName[expectedPort.Name] status := domain.RuntimePortStatus{ @@ -147,6 +152,14 @@ func (b *Backend) RoutingTargets(root string, commands map[string]*domain.Comman } func (b *Backend) ensureExpectedServices(ctx context.Context, root string, commandName string, procedureName string, procedure *domain.Procedure, ports []domain.Port) error { + return b.ensureExpectedServicesWithOptions(ctx, root, commandName, procedureName, procedure, ports, false) +} + +func (b *Backend) ensurePersistentExpectedServices(ctx context.Context, root string, commandName string, procedureName string, procedure *domain.Procedure, ports []domain.Port) error { + return b.ensureExpectedServicesWithOptions(ctx, root, commandName, procedureName, procedure, ports, true) +} + +func (b *Backend) ensureExpectedServicesWithOptions(ctx context.Context, root string, commandName string, procedureName string, procedure *domain.Procedure, ports []domain.Port, publishNotReady bool) error { namespace, _, err := parseRef(root) if err != nil { logger.Log().Error("Cannot reconcile Kubernetes Services for invalid root", zap.String("root", root), zap.String("command", commandName), zap.String("procedure", procedureName), zap.Error(err)) @@ -160,6 +173,7 @@ func (b *Backend) ensureExpectedServices(ctx context.Context, root string, comma logger.Log().Error("Failed to build Kubernetes Service for expected port", zap.String("namespace", namespace), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("port", expected.Name), zap.Error(err)) return err } + service.Spec.PublishNotReadyAddresses = publishNotReady logger.Log().Debug("Reconciling Kubernetes expected-port Service", zap.String("namespace", namespace), zap.String("command", commandName), diff --git a/internal/runtime/kubernetes/procedure_attempts.go b/internal/runtime/kubernetes/procedure_attempts.go index f51ecb2d..ae97b4c5 100644 --- a/internal/runtime/kubernetes/procedure_attempts.go +++ b/internal/runtime/kubernetes/procedure_attempts.go @@ -101,16 +101,13 @@ func (b *Backend) createOrReuseProcedureJob(ctx context.Context, namespace strin var active *batchv1.Job for idx := range jobs.Items { job := &jobs.Items[idx] - if job.Status.Succeeded > 0 { - if err := b.deleteJobAndWait(ctx, namespace, job.Name); err != nil { - return nil, err - } - continue - } attempt := procedureJobAttempt(job, baseName) if attempt >= nextAttempt { nextAttempt = attempt + 1 } + if job.Status.Succeeded > 0 { + continue + } if kubernetesJobFailed(job) { logger.Log().Warn("Retaining failed Kubernetes procedure job for retry", zap.String("namespace", namespace), diff --git a/internal/runtime/kubernetes/procedures.go b/internal/runtime/kubernetes/procedures.go index c601d426..5a41f789 100644 --- a/internal/runtime/kubernetes/procedures.go +++ b/internal/runtime/kubernetes/procedures.go @@ -6,7 +6,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" @@ -27,6 +26,9 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { zap.String("root", command.Root), zap.Int("procedures", len(command.Command.Procedures)), ) + if command.Command.Run == domain.RunModePersistent { + return nil, b.runPersistentCommand(command) + } startIndex := 0 if command.Command.Run == domain.RunModeRestart { resumeIndex, err := b.resumeRestartProcedureIndex(context.Background(), command.Root, command.Name, command.Command) @@ -81,31 +83,6 @@ func (b *Backend) RunCommand(command ports.RuntimeCommand) (*int, error) { zap.Bool("signal", procedure.IsSignal()), zap.Int("expected_ports", len(procedure.ExpectedPorts)), ) - if command.Command.Run == domain.RunModePersistent { - if procedure.IsSignal() { - b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) - if err := b.Signal(procedureName, procedure.Target, procedure.Signal, command.Root); err != nil { - b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) - logger.Log().Error("Kubernetes signal procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal), zap.Error(err)) - return nil, err - } - b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusDone, nil) - logger.Log().Info("Kubernetes signal procedure completed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.String("target", procedure.Target), zap.String("signal", procedure.Signal)) - continue - } - if procedure.Image == "" { - err := fmt.Errorf("kubernetes procedure %s requires image", procedureName) - logger.Log().Error("Kubernetes persistent procedure missing image", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Error(err)) - return nil, err - } - b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) - if err := b.ensurePersistentProcedure(context.Background(), command.ScrollID, command.Root, command.Name, procedureName, resourceName, procedure, command.Ports, env); err != nil { - b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) - logger.Log().Error("Kubernetes persistent procedure failed", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("procedure", procedureName), zap.Error(err)) - return nil, err - } - continue - } b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) exitCode, err := b.runJobProcedure(command.ScrollID, command.Name, procedureName, resourceName, procedure, command.Root, command.Ports, env) if err != nil { @@ -196,34 +173,9 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure logger.Log().Error("Failed to create Kubernetes job procedure", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("base_job", resourceName), zap.Error(err)) return nil, err } - output := make(chan string, 100) - consoleID := runtimeConsoleID(scrollID, procedureName) - console, doneChan := b.consoleManager.AddConsoleWithChannel(consoleID, domain.ConsoleTypeContainer, "stdin", output) - console.WriteInput = func(data string) error { - return b.attachToProcedure(root, procedureName, data) - } - streamStarted := false jobName := createdJob.Name - podName, err := b.waitForJobPod(ctx, namespace, jobName, string(createdJob.UID)) - if err == nil { - streamStarted = true - logger.Log().Debug("Streaming Kubernetes job procedure logs", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.String("pod", podName), zap.String("console_id", consoleID)) - go b.streamPodLogs(ctx, namespace, podName, output) - } else { - logger.Log().Warn("Could not find Kubernetes job pod before wait; console logs may be empty", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.Error(err)) - } exitCode, err := b.waitForJobWithIdleStop(ctx, namespace, jobName, b.keepAliveTrafficIdleStopper(namespace, root, commandName, procedureName, procedure, ports)) - if exitCode != nil { - console.MarkExited(*exitCode) - } - if !streamStarted { - close(output) - } - <-doneChan if err != nil { - if exitCode != nil && *exitCode == 0 { - b.deleteFinishedJob(context.Background(), namespace, jobName) - } if exitCode != nil && *exitCode != 0 { logger.Log().Warn("Keeping failed Kubernetes job procedure for debugging", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) } @@ -231,7 +183,6 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure return exitCode, err } if exitCode != nil && *exitCode == 0 { - b.deleteFinishedJob(context.Background(), namespace, jobName) logger.Log().Info("Kubernetes job procedure exited", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) } else if exitCode != nil { logger.Log().Warn("Keeping failed Kubernetes job procedure for debugging", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) @@ -239,84 +190,92 @@ func (b *Backend) runJobProcedure(scrollID string, commandName string, procedure return exitCode, nil } -func (b *Backend) ensurePersistentProcedure(ctx context.Context, scrollID string, root string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, ports []domain.Port, env map[string]string) error { - if err := b.ensureExpectedServices(ctx, root, commandName, procedureName, procedure, ports); err != nil { - logger.Log().Error("Failed to reconcile Kubernetes persistent procedure Services", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.Error(err)) - return err +func (b *Backend) runPersistentCommand(command ports.RuntimeCommand) error { + ctx := context.Background() + last := -1 + for idx, procedure := range command.Command.Procedures { + if procedure == nil { + continue + } + procedureName := domain.ProcedureName(command.Name, idx, procedure) + if procedure.IsSignal() || procedure.Image == "" { + err := fmt.Errorf("kubernetes persistent procedure %s must be a container with an image", procedureName) + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) + return err + } + if err := b.ensurePersistentExpectedServices(ctx, command.Root, command.Name, procedureName, procedure, command.Ports); err != nil { + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusError, nil) + return err + } + b.observeProcedureStatus(command, procedureName, domain.ScrollLockStatusRunning, nil) + last = idx + } + if last < 0 { + return fmt.Errorf("kubernetes persistent command %s requires at least one procedure", command.Name) } - namespace, pvc, err := parseRef(root) + mainProcedure := command.Command.Procedures[last] + mainName := domain.ProcedureName(command.Name, last, mainProcedure) + resourceName := procedureResourceName(command.Root, command.Name, last) + namespace, pvc, err := parseRef(command.Root) if err != nil { - logger.Log().Error("Kubernetes persistent procedure root ref invalid", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("root", root), zap.Error(err)) + logger.Log().Error("Kubernetes persistent command root ref invalid", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("root", command.Root), zap.Error(err)) return err } - statefulSet, err := procedureStatefulSetSpec(namespace, root, commandName, procedureName, resourceName, procedure, env, b.config.RegistrySecret) + statefulSet, err := persistentStatefulSetSpec(namespace, command.Root, command.Name, resourceName, command.Command, command.ProcedureEnv, b.config.RegistrySecret) if err != nil { - logger.Log().Error("Failed to build Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.Error(err)) + logger.Log().Error("Failed to build Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.Error(err)) return err } if err := b.pinPodToRuntimeNode(ctx, namespace, pvc, &statefulSet.Spec.Template.Spec); err != nil { return err } - logger.Log().Info("Reconciling Kubernetes persistent procedure", - zap.String("scroll_id", scrollID), - zap.String("command", commandName), - zap.String("procedure", procedureName), + if err := b.deleteLegacyPersistentStatefulSets(ctx, namespace, pvc, command.Name, statefulSet.Name); err != nil { + return err + } + logger.Log().Info("Reconciling Kubernetes persistent command", + zap.String("scroll_id", command.ScrollID), + zap.String("command", command.Name), + zap.String("main_procedure", mainName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), - ) - logger.Log().Debug("Kubernetes persistent procedure details", - zap.String("scroll_id", scrollID), - zap.String("command", commandName), - zap.String("procedure", procedureName), - zap.String("resource", resourceName), - zap.String("image", procedure.Image), - zap.Int("env_count", len(env)), - zap.Int("expected_ports", len(procedure.ExpectedPorts)), - zap.Int("mounts", len(procedure.Mounts)), + zap.Int("init_containers", len(statefulSet.Spec.Template.Spec.InitContainers)), ) existing, err := b.client.AppsV1().StatefulSets(namespace).Get(ctx, statefulSet.Name, metav1.GetOptions{}) switch { case apierrors.IsNotFound(err): - logger.Log().Info("Creating Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name)) + logger.Log().Info("Creating Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name)) if _, err := b.client.AppsV1().StatefulSets(namespace).Create(ctx, statefulSet, metav1.CreateOptions{}); err != nil { - logger.Log().Error("Failed to create Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) + logger.Log().Error("Failed to create Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) return err } case err != nil: - logger.Log().Error("Failed to get Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) + logger.Log().Error("Failed to get Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) return err default: - logger.Log().Info("Updating Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.String("resource_version", existing.ResourceVersion)) + logger.Log().Info("Updating Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.String("resource_version", existing.ResourceVersion)) statefulSet.ResourceVersion = existing.ResourceVersion if _, err := b.client.AppsV1().StatefulSets(namespace).Update(ctx, statefulSet, metav1.UpdateOptions{}); err != nil { - logger.Log().Error("Failed to update Kubernetes persistent procedure StatefulSet", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) + logger.Log().Error("Failed to update Kubernetes persistent command StatefulSet", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) return err } } - output := make(chan string, 100) - console, _ := b.consoleManager.AddConsoleWithChannel(runtimeConsoleID(scrollID, procedureName), domain.ConsoleTypeContainer, "stdin", output) - console.WriteInput = func(data string) error { - return b.attachToProcedure(root, procedureName, data) - } - if err := b.waitForStatefulSet(ctx, namespace, statefulSet.Name); err != nil { - close(output) - logger.Log().Error("Kubernetes persistent procedure did not become ready", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) + logger.Log().Info("Kubernetes persistent command started", zap.String("scroll_id", command.ScrollID), zap.String("command", command.Name), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name)) + return nil +} + +func (b *Backend) deleteLegacyPersistentStatefulSets(ctx context.Context, namespace string, pvc string, commandName string, keepName string) error { + selector := fmt.Sprintf("%s=%s,%s=%s", labelScrollID, dnsLabel(pvc), labelCommand, dnsLabel(commandName)) + statefulSets, err := b.client.AppsV1().StatefulSets(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { return err } - logger.Log().Info("Kubernetes persistent procedure ready", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name)) - go func() { - podName, err := b.waitForPodBySelector(context.Background(), namespace, labels.SelectorFromSet(labels.Set{ - labelScrollID: statefulSet.Labels[labelScrollID], - labelProcedure: statefulSet.Labels[labelProcedure], - }).String()) - if err != nil { - logger.Log().Warn("Failed to find Kubernetes persistent procedure pod for logs", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("statefulset", statefulSet.Name), zap.Error(err)) - output <- fmt.Sprintf("failed to find StatefulSet pod logs: %v", err) - close(output) - return + for _, statefulSet := range statefulSets.Items { + if statefulSet.Name == keepName { + continue } - logger.Log().Debug("Streaming Kubernetes persistent procedure logs", zap.String("scroll_id", scrollID), zap.String("command", commandName), zap.String("procedure", procedureName), zap.String("namespace", namespace), zap.String("pod", podName)) - b.streamPodLogs(context.Background(), namespace, podName, output) - }() + if err := b.client.AppsV1().StatefulSets(namespace).Delete(ctx, statefulSet.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } return nil } diff --git a/internal/runtime/kubernetes/resources.go b/internal/runtime/kubernetes/resources.go index e9a0718c..5949af75 100644 --- a/internal/runtime/kubernetes/resources.go +++ b/internal/runtime/kubernetes/resources.go @@ -61,8 +61,14 @@ chown -R 1000:1000 "$DRUID_WORKER_ROOT"` const ( runtimeWorkerServiceAccount = "druid-runtime-worker" workloadTokenPath = "/var/run/secrets/druid-cli/token" + finishedJobTTLSeconds = int32(24 * 60 * 60) ) +func finishedJobTTL() *int32 { + ttl := finishedJobTTLSeconds + return &ttl +} + func workloadTokenVolume(audience string) ([]corev1.Volume, []corev1.VolumeMount) { return []corev1.Volume{{Name: "druid-cli-token", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: audience, ExpirationSeconds: ptrInt64(600), Path: "token"}}}}}}}, []corev1.VolumeMount{{Name: "druid-cli-token", MountPath: "/var/run/secrets/druid-cli", ReadOnly: true}} } @@ -169,7 +175,8 @@ func helperJobSpec(namespace string, jobName string, pvc string, image string, c return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: jobName, Namespace: namespace, Labels: allLabels}, Spec: batchv1.JobSpec{ - BackoffLimit: &backoff, + BackoffLimit: &backoff, + TTLSecondsAfterFinished: finishedJobTTL(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: allLabels}, Spec: podSpec, @@ -221,7 +228,8 @@ func procedureJobSpec(namespace string, root string, commandName string, procedu Labels: labels, }, Spec: batchv1.JobSpec{ - BackoffLimit: &backoff, + BackoffLimit: &backoff, + TTLSecondsAfterFinished: finishedJobTTL(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: podSpec, @@ -231,36 +239,92 @@ func procedureJobSpec(namespace string, root string, commandName string, procedu } func procedureStatefulSetSpec(namespace string, root string, commandName string, procedureName string, resourceName string, procedure *domain.Procedure, env map[string]string, registrySecret string) (*appsv1.StatefulSet, error) { + procedureCopy := *procedure + procedureCopy.Id = &procedureName + command := &domain.CommandInstructionSet{Procedures: []*domain.Procedure{&procedureCopy}} + return persistentStatefulSetSpec(namespace, root, commandName, resourceName, command, map[string]map[string]string{procedureName: env}, registrySecret) +} + +func persistentStatefulSetSpec(namespace string, root string, commandName string, resourceName string, command *domain.CommandInstructionSet, procedureEnv map[string]map[string]string, registrySecret string) (*appsv1.StatefulSet, error) { _, pvc, err := parseRef(root) if err != nil { return nil, err } + if command == nil || len(command.Procedures) == 0 { + return nil, fmt.Errorf("persistent command %s requires at least one procedure", commandName) + } + + last := len(command.Procedures) - 1 + for last >= 0 && command.Procedures[last] == nil { + last-- + } + if last < 0 { + return nil, fmt.Errorf("persistent command %s requires at least one container procedure", commandName) + } + mainProcedure := command.Procedures[last] + if mainProcedure.IsSignal() || mainProcedure.Image == "" { + return nil, fmt.Errorf("persistent command %s final procedure must be a container with an image", commandName) + } + mainName := domain.ProcedureName(commandName, last, mainProcedure) + selectorLabels := baseLabels(pvc) selectorLabels[labelRuntimeID] = dnsLabel(runtimeID(root)) - selectorLabels[labelProcedure] = dnsLabel(procedureName) + selectorLabels[labelProcedure] = dnsLabel(mainName) selectorLabels[labelCommand] = dnsLabel(commandName) labels := copyLabels(selectorLabels) - for _, expectedPort := range procedure.ExpectedPorts { - labels[portSelectorLabel(expectedPort.Name)] = "true" - } - if len(procedure.ExpectedPorts) == 1 { - labels[labelPortName] = dnsLabel(procedure.ExpectedPorts[0].Name) + for _, procedure := range command.Procedures { + if procedure == nil { + continue + } + for _, expectedPort := range procedure.ExpectedPorts { + labels[portSelectorLabel(expectedPort.Name)] = "true" + } } replicas := int32(1) + mainEnv := procedureEnv[mainName] + if mainEnv == nil { + mainEnv = mainProcedure.Env + } container := corev1.Container{ Name: "main", - Image: procedure.Image, - Command: procedure.Command, - WorkingDir: procedure.WorkingDir, - TTY: procedure.TTY, + Image: mainProcedure.Image, + Command: mainProcedure.Command, + WorkingDir: mainProcedure.WorkingDir, + TTY: mainProcedure.TTY, Stdin: true, ImagePullPolicy: corev1.PullIfNotPresent, - Env: envVars(env), - VolumeMounts: volumeMounts(procedure.Mounts), + Env: envVars(mainEnv), + VolumeMounts: volumeMounts(mainProcedure.Mounts), + } + initContainers := make([]corev1.Container, 0, last) + for idx, procedure := range command.Procedures[:last] { + if procedure == nil { + continue + } + procedureName := domain.ProcedureName(commandName, idx, procedure) + if procedure.IsSignal() || procedure.Image == "" { + return nil, fmt.Errorf("persistent command %s init procedure %s must be a container with an image", commandName, procedureName) + } + env := procedureEnv[procedureName] + if env == nil { + env = procedure.Env + } + initContainers = append(initContainers, corev1.Container{ + Name: persistentInitContainerName(procedureName), + Image: procedure.Image, + Command: procedure.Command, + WorkingDir: procedure.WorkingDir, + TTY: procedure.TTY, + Stdin: true, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: envVars(env), + VolumeMounts: volumeMounts(procedure.Mounts), + }) } podSpec := corev1.PodSpec{ - Containers: []corev1.Container{container}, - Volumes: []corev1.Volume{pvcVolume("data", pvc)}, + InitContainers: initContainers, + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{pvcVolume("data", pvc)}, } if registrySecret != "" { podSpec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: registrySecret}} @@ -283,6 +347,10 @@ func procedureStatefulSetSpec(namespace string, root string, commandName string, }, nil } +func persistentInitContainerName(procedureName string) string { + return dnsLabel("init-" + procedureName) +} + func serviceSpec(namespace string, root string, selector map[string]string, portName string, port domain.Port) (*corev1.Service, error) { _, pvc, err := parseRef(root) if err != nil { diff --git a/internal/runtime/kubernetes/resources_test.go b/internal/runtime/kubernetes/resources_test.go index 9b5cfa91..a9584665 100644 --- a/internal/runtime/kubernetes/resources_test.go +++ b/internal/runtime/kubernetes/resources_test.go @@ -20,7 +20,6 @@ import ( "github.com/highcard-dev/daemon/internal/core/domain" "github.com/highcard-dev/daemon/internal/core/ports" - coreservices "github.com/highcard-dev/daemon/internal/core/services" ) func setStatsReader(backend *Backend, nodeName string, summary *nodeStatsSummary, err error) { @@ -35,6 +34,13 @@ func setStatsReader(backend *Backend, nodeName string, summary *nodeStatsSummary } } +func assertFinishedJobTTL(t *testing.T, job *batchv1.Job) { + t.Helper() + if job.Spec.TTLSecondsAfterFinished == nil || *job.Spec.TTLSecondsAfterFinished != finishedJobTTLSeconds { + t.Fatalf("TTLSecondsAfterFinished = %#v, want %d", job.Spec.TTLSecondsAfterFinished, finishedJobTTLSeconds) + } +} + func podStats(namespace string, uid string, rx uint64, tx uint64) *nodeStatsSummary { rxCopy := rx txCopy := tx @@ -74,7 +80,7 @@ func ptrString(value string) *string { } func TestRootRefUsesRequestedNamespace(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) if got, want := backend.RootRef("deployment-123", "games"), ref("games", dataPVCName("deployment-123")); got != want { t.Fatalf("RootRef = %s, want %s", got, want) } @@ -103,6 +109,7 @@ func TestProcedureJobSpecBuildsDeterministicMountsAndLabels(t *testing.T) { if job.Namespace != "druid" { t.Fatalf("namespace = %s, want druid", job.Namespace) } + assertFinishedJobTTL(t, job) if job.Labels[labelManagedBy] != "druid" || job.Labels[labelProcedure] != "start" { t.Fatalf("labels = %#v", job.Labels) } @@ -127,7 +134,7 @@ func TestPinPodToRuntimeNodeUsesRunningPVCConsumer(t *testing.T) { runtimePod := runningProcedurePod("druid", root, "start", "start", 1, "runtime-pod", "start-job") runtimePod.Spec.Volumes = []corev1.Volume{pvcVolume("data", "druid-static-web-data")} client := fake.NewSimpleClientset(runtimePod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) podSpec := corev1.PodSpec{} if err := backend.pinPodToRuntimeNode(context.Background(), "druid", "druid-static-web-data", &podSpec); err != nil { @@ -147,7 +154,7 @@ func TestPinPodToRuntimeNodeIgnoresInactiveOrUnrelatedPods(t *testing.T) { unrelated.Name = "unrelated-pod" unrelated.Spec.Volumes = []corev1.Volume{pvcVolume("data", "another-pvc")} client := fake.NewSimpleClientset(inactive, unrelated) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) podSpec := corev1.PodSpec{} if err := backend.pinPodToRuntimeNode(context.Background(), "druid", "druid-static-web-data", &podSpec); err != nil { @@ -163,7 +170,7 @@ func TestCreateOrReuseProcedureJobPinsToRuntimePVCNode(t *testing.T) { runtimePod := runningProcedurePod("druid", root, "start", "start", 1, "runtime-pod", "start-job") runtimePod.Spec.Volumes = []corev1.Volume{pvcVolume("data", "druid-static-web-data")} client := fake.NewSimpleClientset(runtimePod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) procedure := &domain.Procedure{Image: "alpine:3.20", Command: []string{"true"}} job, err := backend.createOrReuseProcedureJob(context.Background(), "druid", root, "build", "build", "build-job", procedure, nil) @@ -282,6 +289,84 @@ func TestProcedureStatefulSetSpecBuildsPersistentWorkload(t *testing.T) { } } +func TestPersistentStatefulSetRunsEarlierProceduresAsInitContainers(t *testing.T) { + root := ref("druid", "druid-minecraft-data") + command := &domain.CommandInstructionSet{Run: domain.RunModePersistent, Procedures: []*domain.Procedure{ + { + Id: ptrString("coldstart"), + Image: "druid:local", + Command: []string{"druid-coldstarter"}, + ExpectedPorts: []domain.ExpectedPort{{Name: "main"}, {Name: "rcon"}}, + Mounts: []domain.Mount{{Path: "/runtime", SubPath: "."}}, + }, + { + Id: ptrString("start"), + Image: "eclipse-temurin:21-jre", + Command: []string{"sh", "./start.sh"}, + ExpectedPorts: []domain.ExpectedPort{{Name: "main"}, {Name: "rcon"}}, + Mounts: []domain.Mount{{Path: "/server"}}, + }, + }} + statefulSet, err := persistentStatefulSetSpec("druid", root, "start", "minecraft-start-1", command, map[string]map[string]string{ + "coldstart": {"DRUID_PORT_MAIN": "25565"}, + "start": {"DRUID_PORT_MAIN": "25565"}, + }, "registry-secret") + if err != nil { + t.Fatal(err) + } + pod := statefulSet.Spec.Template.Spec + if len(pod.InitContainers) != 1 || pod.InitContainers[0].Name != "init-coldstart" || pod.InitContainers[0].Image != "druid:local" { + t.Fatalf("init containers = %#v", pod.InitContainers) + } + if len(pod.Containers) != 1 || pod.Containers[0].Name != "main" || pod.Containers[0].Image != "eclipse-temurin:21-jre" { + t.Fatalf("containers = %#v", pod.Containers) + } + if statefulSet.Spec.Selector.MatchLabels[labelProcedure] != "start" { + t.Fatalf("selector = %#v", statefulSet.Spec.Selector.MatchLabels) + } + for _, portName := range []string{"main", "rcon"} { + if statefulSet.Spec.Template.Labels[portSelectorLabel(portName)] != "true" { + t.Fatalf("pod labels = %#v, want port %s", statefulSet.Spec.Template.Labels, portName) + } + } +} + +func TestRunPersistentCommandCreatesOneStatefulSetAndInitReadyServices(t *testing.T) { + root := ref("druid", "druid-minecraft-data") + legacyLabels := baseLabels("druid-minecraft-data") + legacyLabels[labelCommand] = "start" + legacyLabels[labelProcedure] = "coldstart" + client := fake.NewSimpleClientset(&appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "minecraft-start-0", Namespace: "druid", Labels: legacyLabels}}) + backend := NewWithClient(Config{Namespace: "druid"}, client) + command := &domain.CommandInstructionSet{Run: domain.RunModePersistent, Procedures: []*domain.Procedure{ + {Id: ptrString("coldstart"), Image: "druid:local", ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, + {Id: ptrString("start"), Image: "minecraft:local", ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}}, + }} + if _, err := backend.RunCommand(ports.RuntimeCommand{ + ScrollID: "minecraft", + Root: root, + Name: "start", + Command: command, + Ports: []domain.Port{{Name: "main", Port: 25565, Protocol: "tcp"}}, + }); err != nil { + t.Fatal(err) + } + statefulSets, err := client.AppsV1().StatefulSets("druid").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(statefulSets.Items) != 1 || statefulSets.Items[0].Name != "minecraft-start-1" || len(statefulSets.Items[0].Spec.Template.Spec.InitContainers) != 1 { + t.Fatalf("StatefulSets = %#v", statefulSets.Items) + } + service, err := client.CoreV1().Services("druid").Get(context.Background(), portServiceName(root, "main"), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !service.Spec.PublishNotReadyAddresses { + t.Fatal("persistent port Service must publish the init-running Pod") + } +} + func TestProcedureWorkloadsLabelEveryExpectedPort(t *testing.T) { procedure := &domain.Procedure{ Image: "nginx:1.27", @@ -323,6 +408,7 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { TokenFile: "token-file", } job := workerPullJobSpec("druid", "worker-pull", "runtime-pvc", "druid-cli:test", action, "pull-secret", "runtime-registry", true, "druid-cli") + assertFinishedJobTTL(t, job) container := job.Spec.Template.Spec.Containers[0] command := strings.Join(container.Command, " ") for _, want := range []string{"druid --config /tmp/druid-registry.json", "worker pull", "--mode update", "--runtime-id deployment-123", "--callback-url", "chown -R 1000:1000"} { @@ -359,11 +445,11 @@ func TestWorkerPullJobSpecRunsDruidWorkerPull(t *testing.T) { func TestSpawnPullWorkerCreateUsesFinalPVCAndWorkerJob(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, client) var jobs []*batchv1.Job - backend.jobLogRunner = func(ctx context.Context, job *batchv1.Job) ([]byte, error) { + backend.helperJobRunner = func(ctx context.Context, job *batchv1.Job) error { jobs = append(jobs, job.DeepCopy()) - return nil, nil + return nil } action := ports.RuntimeWorkerAction{ Mode: ports.RuntimeWorkerModeCreate, @@ -375,7 +461,11 @@ func TestSpawnPullWorkerCreateUsesFinalPVCAndWorkerJob(t *testing.T) { CallbackURL: "http://druid-cli:8083/internal/v1/workers/deployment-123/complete", TokenFile: "token-file", } - if err := backend.SpawnPullWorker(context.Background(), action); err != nil { + done, err := backend.SpawnPullWorker(context.Background(), action) + if err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { t.Fatal(err) } pvcs, err := client.CoreV1().PersistentVolumeClaims("games").List(context.Background(), metav1.ListOptions{}) @@ -411,7 +501,7 @@ func TestDeleteFinishedJobRemovesJob(t *testing.T) { client := fake.NewSimpleClientset(&batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: "finished", Namespace: "druid"}, }) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) backend.deleteFinishedJob(context.Background(), "druid", "finished") @@ -428,7 +518,7 @@ func TestCreateFreshJobKeepsFailedJob(t *testing.T) { Status: corev1.ConditionTrue, }}}, }) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) created, err := backend.createFreshJob(context.Background(), &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "failed", Namespace: "druid"}}) if err != nil { @@ -459,7 +549,7 @@ func TestCreateOrReuseProcedureJobRetainsFailedBaseAndCreatesRetry(t *testing.T) Status: corev1.ConditionTrue, }}}, }) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) created, err := backend.createOrReuseProcedureJob(context.Background(), "druid", root, "start", "start.1", base, &domain.Procedure{Image: "alpine"}, nil) if err != nil { @@ -480,7 +570,7 @@ func TestCreateOrReuseProcedureJobUsesNextRetryAttempt(t *testing.T) { failedProcedureJob(root, base, "start", "start.1", 1), failedProcedureJob(root, base+"-r2", "start", "start.1", 2), ) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) created, err := backend.createOrReuseProcedureJob(context.Background(), "druid", root, "start", "start.1", base, &domain.Procedure{Image: "alpine"}, nil) if err != nil { @@ -503,7 +593,7 @@ func TestCreateOrReuseProcedureJobReusesActiveAttempt(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(active) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) created, err := backend.createOrReuseProcedureJob(context.Background(), "druid", root, "start", "coldstart", base, &domain.Procedure{Image: "alpine"}, nil) if err != nil { @@ -542,7 +632,7 @@ func TestResumeRestartProcedureIndexDeletesSupersededActiveProcedure(t *testing. Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(coldstartJob, startJob) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) resumeIndex, err := backend.resumeRestartProcedureIndex(context.Background(), root, "start", &domain.CommandInstructionSet{Procedures: []*domain.Procedure{ {Id: &coldstart}, @@ -574,7 +664,7 @@ func TestResumeRestartProcedureIndexKeepsOnlyActiveFirstProcedure(t *testing.T) Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(coldstartJob) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) resumeIndex, err := backend.resumeRestartProcedureIndex(context.Background(), root, "start", &domain.CommandInstructionSet{Procedures: []*domain.Procedure{{Id: &coldstart}}}) if err != nil { @@ -588,7 +678,7 @@ func TestResumeRestartProcedureIndexKeepsOnlyActiveFirstProcedure(t *testing.T) } } -func TestCreateOrReuseProcedureJobDeletesSucceededAttempt(t *testing.T) { +func TestCreateOrReuseProcedureJobRetainsSucceededAttempt(t *testing.T) { root := ref("druid", dataPVCName("deployment-123")) base := procedureResourceName(root, "install", 0) client := fake.NewSimpleClientset(&batchv1.Job{ @@ -599,14 +689,17 @@ func TestCreateOrReuseProcedureJobDeletesSucceededAttempt(t *testing.T) { }, Status: batchv1.JobStatus{Succeeded: 1}, }) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) created, err := backend.createOrReuseProcedureJob(context.Background(), "druid", root, "install", "install", base, &domain.Procedure{Image: "alpine"}, nil) if err != nil { t.Fatal(err) } - if created.Name != base { - t.Fatalf("job name = %s, want clean base %s after deleting succeeded attempt", created.Name, base) + if want := procedureAttemptName(base, 2); created.Name != want { + t.Fatalf("job name = %s, want %s", created.Name, want) + } + if _, err := client.BatchV1().Jobs("druid").Get(context.Background(), base, metav1.GetOptions{}); err != nil { + t.Fatalf("retained succeeded job: %v", err) } } @@ -628,7 +721,7 @@ func TestWaitForJobReportsPodFailureReason(t *testing.T) { }}, }}}, } - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset(job, pod)) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset(job, pod)) exitCode, err := backend.waitForJob(context.Background(), "druid", "failed-start") if err == nil { @@ -642,8 +735,27 @@ func TestWaitForJobReportsPodFailureReason(t *testing.T) { } } +func TestWaitForJobImmediatelyReportsUnschedulablePod(t *testing.T) { + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "unschedulable", Namespace: "druid"}} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "unschedulable-pod", Namespace: "druid", Labels: map[string]string{"job-name": job.Name}}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "insufficient memory", + }}}, + } + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset(job, pod)) + + exitCode, err := backend.waitForJob(context.Background(), "druid", job.Name) + if err == nil || !strings.Contains(err.Error(), "insufficient memory") { + t.Fatalf("waitForJob error = %v, want unschedulable detail", err) + } + if exitCode == nil || *exitCode != 1 { + t.Fatalf("exitCode = %#v, want 1", exitCode) + } +} + func TestWaitForJobUsesRecentSuccessfulDeletion(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) backend.recordJobExit("druid", "finished", 0) exitCode, err := backend.waitForJob(context.Background(), "druid", "finished") @@ -656,7 +768,7 @@ func TestWaitForJobUsesRecentSuccessfulDeletion(t *testing.T) { } func TestWaitForJobMissingWithoutRecentExitFails(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) if _, err := backend.waitForJob(context.Background(), "druid", "missing"); !apierrors.IsNotFound(err) { t.Fatalf("waitForJob error = %v, want not found", err) @@ -734,7 +846,7 @@ func TestStartupContainerFailureReportsNonzeroTermination(t *testing.T) { func TestExpectedServicesUseRootNamespace(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid-system"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid-system"}, client) root := ref("games", dataPVCName("deployment-123")) procedure := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "http"}}} @@ -752,7 +864,7 @@ func TestExpectedServicesUseRootNamespace(t *testing.T) { func TestRegistryConfigSecretUsesDruidClientConfigShape(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) secretName, cleanup, err := backend.createRegistryConfigSecret(context.Background(), "druid", "artifact", []domain.RegistryCredential{{ Host: "artifacts.druid.gg/user/scroll", Username: "robot$scroll", @@ -779,7 +891,7 @@ func TestRegistryConfigSecretUsesDruidClientConfigShape(t *testing.T) { func TestExpectedPortsUsesPodStatsTraffic(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") procedureName := "start" pod := runningProcedurePod("druid", root, "start", procedureName, 1, "pod-start-stats", "start-job") @@ -835,7 +947,7 @@ func TestExpectedPortsUsesPodStatsTraffic(t *testing.T) { func TestExpectedPortsDegradesWhenPodStatsUnavailable(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") pod := runningProcedurePod("druid", root, "start", "start", 1, "pod-start-unavailable", "start-job") if _, err := client.CoreV1().Pods("druid").Create(context.Background(), pod, metav1.CreateOptions{}); err != nil { @@ -882,7 +994,7 @@ func TestExpectedPortsDegradesWhenPodStatsUnavailable(t *testing.T) { func TestExpectedPortsWithoutActivePodDoesNotBorrowTraffic(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") procedureName := "start" unrelated := runningProcedurePod("druid", ref("druid", "other-scroll"), "start", procedureName, 1, "pod-other", "start-job") @@ -945,7 +1057,7 @@ func TestKeepAliveTrafficStopsIdleRunningProcedure(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(job, pod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) globalPodTrafficStore.record("pod-idle-stop", 100, 50, time.Now().Add(-2*time.Minute)) setStatsReader(backend, "node-a", podStats("druid", "pod-idle-stop", 100, 50), nil) stopper := backend.keepAliveTrafficIdleStopper("druid", root, "start", "start", &domain.Procedure{ @@ -980,7 +1092,7 @@ func TestKeepAliveTrafficKeepsProcedureWhenTrafficPresent(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(job, pod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) globalPodTrafficStore.record("pod-traffic-present", 100, 50, time.Now().Add(-2*time.Minute)) globalPodTrafficStore.record("pod-traffic-present", 100, 50, time.Now().Add(-500*time.Millisecond)) setStatsReader(backend, "node-a", podStats("druid", "pod-traffic-present", 102, 55), nil) @@ -1013,7 +1125,7 @@ func TestKeepAliveTrafficKeepsProcedureWhenPodStatsUnavailable(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(job, pod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) setStatsReader(backend, "node-a", nil, errors.New("stats unavailable")) stopper := backend.keepAliveTrafficIdleStopper("druid", root, "start", "start", &domain.Procedure{ ExpectedPorts: []domain.ExpectedPort{{Name: "main", KeepAliveTraffic: "1b/1s"}}, @@ -1044,7 +1156,7 @@ func TestKeepAliveTrafficKeepsProcedureWhilePodStatsWarmUp(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(job, pod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) setStatsReader(backend, "node-a", podStats("druid", "pod-stats-warmup", 100, 50), nil) stopper := backend.keepAliveTrafficIdleStopper("druid", root, "start", "start", &domain.Procedure{ ExpectedPorts: []domain.ExpectedPort{{Name: "main", KeepAliveTraffic: "1b/1s"}}, @@ -1072,7 +1184,7 @@ func TestKeepAliveTrafficWaitsForFullWindowBeforeStopping(t *testing.T) { Status: batchv1.JobStatus{Active: 1}, } client := fake.NewSimpleClientset(job, pod) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) globalPodTrafficStore.record("pod-full-window", 100, 50, time.Now().Add(-2*time.Minute)) setStatsReader(backend, "node-a", podStats("druid", "pod-full-window", 100, 50), nil) stopper := backend.keepAliveTrafficIdleStopper("druid", root, "start", "start", &domain.Procedure{ @@ -1092,7 +1204,7 @@ func TestKeepAliveTrafficWaitsForFullWindowBeforeStopping(t *testing.T) { } func TestKeepAliveTrafficDoesNotStopColdstarter(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) stopper := backend.keepAliveTrafficIdleStopper("druid", ref("druid", "druid-static-web-data"), "start", "coldstart", &domain.Procedure{ Command: []string{"druid-coldstarter"}, ExpectedPorts: []domain.ExpectedPort{{Name: "main", KeepAliveTraffic: "1b/1s"}}, @@ -1103,7 +1215,7 @@ func TestKeepAliveTrafficDoesNotStopColdstarter(t *testing.T) { } func TestRoutingTargetsReturnStableBackendServices(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) root := ref("druid", "druid-static-web-data") procedureID := "web" @@ -1138,7 +1250,7 @@ func TestRoutingTargetsReturnStableBackendServices(t *testing.T) { } func TestRoutingTargetsIncludeAllUnclaimedRuntimePorts(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) root := ref("druid", "druid-static-web-data") ports := []domain.Port{{Name: "vscode", Port: 3333, Protocol: "http"}, {Name: "unused-game-port", Port: 25565, Protocol: "tcp"}} targets, err := backend.RoutingTargets(root, nil, ports) @@ -1157,7 +1269,7 @@ func TestRoutingTargetsIncludeAllUnclaimedRuntimePorts(t *testing.T) { func TestPortRoutingTargetUsesStableServiceCreatedByCommand(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") ports := []domain.Port{{Name: "webdav", Port: 8084, Protocol: "http"}} @@ -1215,7 +1327,7 @@ func TestServiceSpecUsesRuntimePortForServiceAndTarget(t *testing.T) { } func TestRoutingTargetsCollapseColdstarterAndRuntimePort(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) root := ref("druid", "druid-minecraft-data") coldstart := "coldstart" start := "start" @@ -1249,7 +1361,7 @@ func TestRoutingTargetsCollapseColdstarterAndRuntimePort(t *testing.T) { func TestExpectedServiceForSharedPortKeepsStableLabelSelector(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-minecraft-data") coldstart := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}} start := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "main"}}} @@ -1306,7 +1418,7 @@ func TestExpectedServiceReplacesLegacyProcedureNamedService(t *testing.T) { legacyLabels := baseLabels(refPVCName(root)) legacyLabels[labelPortName] = "http" client := fake.NewSimpleClientset(&corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: legacyName, Namespace: "druid", Labels: legacyLabels}}) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) procedure := &domain.Procedure{ExpectedPorts: []domain.ExpectedPort{{Name: "http"}}} if err := backend.ensureExpectedServices(context.Background(), root, "start", "web", procedure, []domain.Port{{Name: "http", Port: 8080, Protocol: "tcp"}}); err != nil { t.Fatal(err) @@ -1320,7 +1432,7 @@ func TestExpectedServiceReplacesLegacyProcedureNamedService(t *testing.T) { } func TestRoutingTargetsUseStablePortSelector(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) root := ref("druid", "druid-minecraft-data") coldstart := "coldstart" start := "start" @@ -1351,7 +1463,7 @@ func TestRoutingTargetsUseStablePortSelector(t *testing.T) { func TestStopRuntimeDeletesWorkloadsButPreservesDataAndServices(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") labels := baseLabels("druid-static-web-data") labels[labelProcedure] = "web" @@ -1415,7 +1527,7 @@ func TestStopCommandPreservesStablePortService(t *testing.T) { t.Fatal(err) } client := fake.NewSimpleClientset(service) - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) if err := backend.StopCommand(root, "start"); err != nil { t.Fatal(err) } @@ -1426,7 +1538,7 @@ func TestStopCommandPreservesStablePortService(t *testing.T) { func TestDeleteRuntimePurgesServicesAndDataWhenRequested(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") service, err := serviceSpec("druid", root, portServiceSelector(refPVCName(root), "http"), "http", domain.Port{Name: "http", Port: 8080, Protocol: "tcp"}) if err != nil { @@ -1453,6 +1565,7 @@ func TestDeleteRuntimePurgesServicesAndDataWhenRequested(t *testing.T) { func TestBackupJobSpecUsesRuntimePVCAndRegistryEnv(t *testing.T) { backup := backupJobSpec("druid", "backup", "runtime-pvc", "druid-cli:test", "registry.local/scroll:backup", "registry-secret", "", true) + assertFinishedJobTTL(t, backup) if backup.Spec.Template.Spec.Containers[0].Command[1] != "push" { t.Fatalf("backup command = %#v", backup.Spec.Template.Spec.Containers[0].Command) } @@ -1468,8 +1581,8 @@ func TestBackupJobSpecUsesRuntimePVCAndRegistryEnv(t *testing.T) { } func TestSpawnPullWorkerRequiresPullImage(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) - err := backend.SpawnPullWorker(context.Background(), ports.RuntimeWorkerAction{ + backend := NewWithClient(Config{Namespace: "druid"}, fake.NewSimpleClientset()) + _, err := backend.SpawnPullWorker(context.Background(), ports.RuntimeWorkerAction{ Mode: ports.RuntimeWorkerModeCreate, RuntimeID: "scroll", Artifact: "ghcr.io/example/scroll:latest", @@ -1484,8 +1597,8 @@ func TestSpawnPullWorkerRequiresPullImage(t *testing.T) { } func TestSpawnPullWorkerRejectsLocalArtifactPath(t *testing.T) { - backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), fake.NewSimpleClientset()) - err := backend.SpawnPullWorker(context.Background(), ports.RuntimeWorkerAction{ + backend := NewWithClient(Config{Namespace: "druid", PullImage: "druid-cli:test"}, fake.NewSimpleClientset()) + _, err := backend.SpawnPullWorker(context.Background(), ports.RuntimeWorkerAction{ Mode: ports.RuntimeWorkerModeCreate, RuntimeID: "scroll", Artifact: t.TempDir(), @@ -1501,7 +1614,7 @@ func TestSpawnPullWorkerRejectsLocalArtifactPath(t *testing.T) { func TestSignalDeletesPersistentStatefulSetAndPods(t *testing.T) { client := fake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "druid"}, coreservices.NewConsoleManager(coreservices.NewLogManager()), client) + backend := NewWithClient(Config{Namespace: "druid"}, client) root := ref("druid", "druid-static-web-data") name := "static-web-start-0" labels := baseLabels("druid-static-web-data") diff --git a/internal/runtime/kubernetes/wait_jobs.go b/internal/runtime/kubernetes/wait_jobs.go index 5e1422d1..64629b5c 100644 --- a/internal/runtime/kubernetes/wait_jobs.go +++ b/internal/runtime/kubernetes/wait_jobs.go @@ -1,10 +1,8 @@ package kubernetes import ( - "bufio" "context" "fmt" - "io" "strings" "time" @@ -116,11 +114,9 @@ func (b *Backend) waitForJobWithIdleStop(ctx context.Context, namespace string, logger.Log().Error("Kubernetes job failed", zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", exitCode), zap.String("detail", detail), zap.Int32("succeeded", job.Status.Succeeded), zap.Int32("failed", job.Status.Failed), zap.Int32("active", job.Status.Active)) return &exitCode, fmt.Errorf("job %s failed: %s", jobName, detail) } - if time.Since(startedAt) > time.Minute { - if exitCode, detail, ok := b.jobStartupFailure(ctx, namespace, jobName); ok { - logger.Log().Error("Kubernetes job startup failed", zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", exitCode), zap.String("detail", detail), zap.Int32("succeeded", job.Status.Succeeded), zap.Int32("failed", job.Status.Failed), zap.Int32("active", job.Status.Active)) - return &exitCode, fmt.Errorf("job %s failed: %s", jobName, detail) - } + if exitCode, detail, ok := b.jobStartupFailure(ctx, namespace, jobName, time.Since(startedAt) > time.Minute); ok { + logger.Log().Error("Kubernetes job startup failed", zap.String("namespace", namespace), zap.String("job", jobName), zap.Int("exit_code", exitCode), zap.String("detail", detail), zap.Int32("succeeded", job.Status.Succeeded), zap.Int32("failed", job.Status.Failed), zap.Int32("active", job.Status.Active)) + return &exitCode, fmt.Errorf("job %s failed: %s", jobName, detail) } if time.Now().After(deadline) { logger.Log().Error("Timed out waiting for Kubernetes job", zap.String("namespace", namespace), zap.String("job", jobName), zap.Int32("succeeded", job.Status.Succeeded), zap.Int32("failed", job.Status.Failed), zap.Int32("active", job.Status.Active)) @@ -146,19 +142,30 @@ func (b *Backend) waitForJobWithIdleStop(ctx context.Context, namespace string, } } -func (b *Backend) jobStartupFailure(ctx context.Context, namespace string, jobName string) (int, string, bool) { +func (b *Backend) jobStartupFailure(ctx context.Context, namespace string, jobName string, includeTerminated bool) (int, string, bool) { selector := labels.SelectorFromSet(labels.Set{"job-name": jobName}).String() pods, err := b.client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) if err != nil || len(pods.Items) == 0 { return 1, "", false } for _, pod := range pods.Items { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == corev1.PodReasonUnschedulable { + return 1, fmt.Sprintf("pod %s is unschedulable: %s", pod.Name, condition.Message), true + } + } for _, status := range pod.Status.InitContainerStatuses { + if status.State.Terminated != nil && !includeTerminated { + continue + } if exitCode, detail, ok := startupContainerFailure(pod.Name, status); ok { return exitCode, detail, true } } for _, status := range pod.Status.ContainerStatuses { + if status.State.Terminated != nil && !includeTerminated { + continue + } if exitCode, detail, ok := startupContainerFailure(pod.Name, status); ok { return exitCode, detail, true } @@ -234,67 +241,3 @@ func startupContainerFailure(podName string, status corev1.ContainerStatus) (int return 1, "", false } } - -func (b *Backend) podLogs(ctx context.Context, namespace string, podName string) ([]byte, error) { - logger.Log().Debug("Reading Kubernetes pod logs", zap.String("namespace", namespace), zap.String("pod", podName)) - req := b.client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{}) - stream, err := req.Stream(ctx) - if err != nil { - logger.Log().Warn("Failed to open Kubernetes pod log stream", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - return nil, err - } - defer stream.Close() - logs, err := io.ReadAll(stream) - if err != nil { - logger.Log().Warn("Failed to read Kubernetes pod logs", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - return logs, err - } - logger.Log().Debug("Read Kubernetes pod logs", zap.String("namespace", namespace), zap.String("pod", podName), zap.Int("bytes", len(logs))) - return logs, nil -} - -func (b *Backend) streamPodLogs(ctx context.Context, namespace string, podName string, output chan<- string) { - defer close(output) - var stream io.ReadCloser - deadline := time.Now().Add(30 * time.Second) - logger.Log().Debug("Opening Kubernetes follow log stream", zap.String("namespace", namespace), zap.String("pod", podName)) - for { - req := b.client.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{Follow: true}) - var err error - stream, err = req.Stream(ctx) - if err == nil { - logger.Log().Debug("Kubernetes follow log stream opened", zap.String("namespace", namespace), zap.String("pod", podName)) - break - } - if !strings.Contains(err.Error(), "ContainerCreating") && - !strings.Contains(err.Error(), "PodInitializing") && - !strings.Contains(err.Error(), "not available") { - logger.Log().Warn("Failed to stream Kubernetes pod logs", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - output <- fmt.Sprintf("failed to stream pod logs: %v", err) - return - } - if time.Now().After(deadline) { - logger.Log().Warn("Timed out opening Kubernetes pod log stream", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - output <- fmt.Sprintf("failed to stream pod logs: %v", err) - return - } - logger.Log().Debug("Kubernetes pod logs not ready yet", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - select { - case <-ctx.Done(): - logger.Log().Warn("Context cancelled while opening Kubernetes pod logs", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(ctx.Err())) - output <- fmt.Sprintf("failed to stream pod logs: %v", ctx.Err()) - return - case <-time.After(500 * time.Millisecond): - } - } - defer stream.Close() - scanner := bufio.NewScanner(stream) - for scanner.Scan() { - output <- scanner.Text() - } - if err := scanner.Err(); err != nil { - logger.Log().Warn("Kubernetes pod log stream ended with scanner error", zap.String("namespace", namespace), zap.String("pod", podName), zap.Error(err)) - return - } - logger.Log().Debug("Kubernetes pod log stream ended", zap.String("namespace", namespace), zap.String("pod", podName)) -} diff --git a/internal/runtime/kubernetes/workers.go b/internal/runtime/kubernetes/workers.go index 354114da..dbe8d68b 100644 --- a/internal/runtime/kubernetes/workers.go +++ b/internal/runtime/kubernetes/workers.go @@ -3,6 +3,7 @@ package kubernetes import ( "context" "encoding/json" + "errors" "fmt" "os" "time" @@ -18,13 +19,13 @@ import ( "go.uber.org/zap" ) -func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) error { +func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { if err := b.config.ValidateForMaterialization(); err != nil { logger.Log().Error("Kubernetes pull worker config invalid", zap.String("runtime_id", action.RuntimeID), zap.Error(err)) - return err + return nil, err } if _, err := os.Stat(action.Artifact); err == nil { - return fmt.Errorf("kubernetes materialization requires an OCI artifact reference; local path %q is only available with Docker runtime", action.Artifact) + return nil, fmt.Errorf("kubernetes materialization requires an OCI artifact reference; local path %q is only available with Docker runtime", action.Artifact) } if action.MountPath == "" { action.MountPath = "/scroll" @@ -35,7 +36,7 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke namespace, pvc, err := parseRef(action.RootRef) if err != nil { logger.Log().Error("Kubernetes pull worker root ref invalid", zap.String("runtime_id", action.RuntimeID), zap.String("root_ref", action.RootRef), zap.Error(err)) - return err + return nil, err } logger.Log().Info("Spawning Kubernetes pull worker", zap.String("runtime_id", action.RuntimeID), @@ -56,32 +57,39 @@ func (b *Backend) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorke // A deployment obtains both workload identities as part of its first // materialization, before creating any runtime resources. if err := b.ensureRuntimeServiceAccounts(ctx, namespace); err != nil { - return err + return nil, err } if action.Mode == ports.RuntimeWorkerModeCreate { if err := b.ensurePVC(ctx, namespace, pvc, action.Storage); err != nil { logger.Log().Error("Failed to ensure runtime PVC for pull worker", zap.String("runtime_id", action.RuntimeID), zap.String("namespace", namespace), zap.String("pvc", pvc), zap.Error(err)) - return err + return nil, err } } registryConfigSecret, cleanupRegistryConfig, err := b.createRegistryConfigSecret(ctx, namespace, action.Artifact+action.RuntimeID, action.RegistryCredentials) if err != nil { logger.Log().Error("Failed to create registry config secret for pull worker", zap.String("runtime_id", action.RuntimeID), zap.String("namespace", namespace), zap.Error(err)) - return err + return nil, err } - defer cleanupRegistryConfig() job := workerPullJobSpec(namespace, jobName("worker-pull", action.RootRef, shortHash(string(action.Mode)+action.Artifact)), pvc, b.config.PullImage, action, b.config.RegistrySecret, registryConfigSecret, b.config.RegistryPlainHTTP, b.config.ServiceAccountAudience) if err := b.pinPodToRuntimeNode(ctx, namespace, pvc, &job.Spec.Template.Spec); err != nil { - return err + cleanupRegistryConfig() + return nil, err } setJobDeadlineFromContext(ctx, job) logger.Log().Debug("Kubernetes pull worker job built", zap.String("runtime_id", action.RuntimeID), zap.String("namespace", namespace), zap.String("job", job.Name)) - if err := b.runHelperJob(ctx, job); err != nil { + done, err := b.startHelperJob(ctx, job) + if err != nil { + cleanupRegistryConfig() logger.Log().Error("Kubernetes pull worker failed", zap.String("runtime_id", action.RuntimeID), zap.String("namespace", namespace), zap.String("job", job.Name), zap.Error(err)) - return err + return nil, err } - logger.Log().Info("Kubernetes pull worker completed", zap.String("runtime_id", action.RuntimeID), zap.String("namespace", namespace), zap.String("job", job.Name)) - return nil + result := make(chan error, 1) + go func() { + err := <-done + cleanupRegistryConfig() + result <- err + }() + return result, nil } func (b *Backend) BackupRuntime(ctx context.Context, root string, artifact string, registryCredentials []domain.RegistryCredential) error { @@ -188,57 +196,42 @@ func (b *Backend) createRegistryConfigSecret(ctx context.Context, namespace stri } func (b *Backend) runHelperJob(ctx context.Context, job *batchv1.Job) error { - _, err := b.runJobAndLogs(ctx, job) + done, err := b.startHelperJob(ctx, job) if err != nil { - logger.Log().Error("Kubernetes helper job failed", zap.String("namespace", job.Namespace), zap.String("job", job.Name), zap.Error(err)) + return err } - return err + return <-done } -func (b *Backend) runJobAndLogs(ctx context.Context, job *batchv1.Job) ([]byte, error) { - if b.jobLogRunner != nil { - logger.Log().Debug("Running Kubernetes job through test log runner", zap.String("namespace", job.Namespace), zap.String("job", job.Name)) - return b.jobLogRunner(ctx, job) +func (b *Backend) startHelperJob(ctx context.Context, job *batchv1.Job) (<-chan error, error) { + done := make(chan error, 1) + if b.helperJobRunner != nil { + go func() { done <- b.helperJobRunner(ctx, job) }() + return done, nil } logger.Log().Info("Starting Kubernetes helper job", zap.String("namespace", job.Namespace), zap.String("job", job.Name)) - logger.Log().Debug("Kubernetes helper job details", zap.String("namespace", job.Namespace), zap.String("job", job.Name), zap.String("service_account", job.Spec.Template.Spec.ServiceAccountName), zap.Int("containers", len(job.Spec.Template.Spec.Containers)), zap.Int("init_containers", len(job.Spec.Template.Spec.InitContainers))) createdJob, err := b.createFreshJob(ctx, job) if err != nil { - logger.Log().Error("Failed to create Kubernetes helper job", zap.String("namespace", job.Namespace), zap.String("job", job.Name), zap.Error(err)) return nil, err } - jobName := createdJob.Name - podName, err := b.waitForJobPod(ctx, job.Namespace, jobName, string(createdJob.UID)) + go func() { done <- b.waitHelperJob(ctx, createdJob) }() + return done, nil +} + +func (b *Backend) waitHelperJob(ctx context.Context, job *batchv1.Job) error { + exitCode, err := b.waitForJob(ctx, job.Namespace, job.Name) if err != nil { - logger.Log().Error("Failed to find Kubernetes helper job pod", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.String("uid", string(createdJob.UID)), zap.Error(err)) - return nil, err - } - logger.Log().Debug("Kubernetes helper job pod found", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.String("pod", podName)) - exitCode, waitErr := b.waitForJob(ctx, job.Namespace, jobName) - logs, logErr := b.podLogs(ctx, job.Namespace, podName) - if logErr != nil { - logger.Log().Warn("Failed to collect Kubernetes helper job logs", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.String("pod", podName), zap.Error(logErr)) - } else { - logger.Log().Debug("Collected Kubernetes helper job logs", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.String("pod", podName), zap.Int("bytes", len(logs))) - } - if exitCode != nil && *exitCode == 0 { - b.deleteFinishedJob(context.Background(), job.Namespace, jobName) - } else if exitCode != nil { - logger.Log().Warn("Keeping failed Kubernetes helper job for debugging", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) - } - if logErr != nil && waitErr == nil { - waitErr = logErr - } - if waitErr != nil { - logger.Log().Error("Kubernetes helper job wait failed", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.Any("exit_code", exitCode), zap.Error(waitErr)) - return logs, waitErr + if errors.Is(err, context.Canceled) { + b.deleteFinishedJob(context.Background(), job.Namespace, job.Name) + } + return err } - if exitCode != nil && *exitCode != 0 { - logger.Log().Error("Kubernetes helper job exited non-zero", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) - return logs, fmt.Errorf("job %s exited with code %d", jobName, *exitCode) + if exitCode == nil { + return fmt.Errorf("job %s completed without an exit code", job.Name) } - if exitCode != nil { - logger.Log().Info("Kubernetes helper job completed", zap.String("namespace", job.Namespace), zap.String("job", jobName), zap.Int("exit_code", *exitCode)) + if *exitCode != 0 { + return fmt.Errorf("job %s exited with code %d", job.Name, *exitCode) } - return logs, nil + b.deleteFinishedJob(context.Background(), job.Namespace, job.Name) + return nil } diff --git a/internal/runtime/kubernetes/workload_identity_test.go b/internal/runtime/kubernetes/workload_identity_test.go index 726b1c17..b40170fa 100644 --- a/internal/runtime/kubernetes/workload_identity_test.go +++ b/internal/runtime/kubernetes/workload_identity_test.go @@ -21,7 +21,7 @@ func TestAuthenticateWorkloadRejectsUnrecognizedServiceAccount(t *testing.T) { Spec: corev1.PodSpec{ServiceAccountName: "unrecognized"}, }) client.PrependReactor("create", "tokenreviews", tokenReviewReactor("unrecognized", "games", "other-0", "pod-uid", true)) - backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, nil, client) + backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, client) if _, err := backend.AuthenticateWorkload(context.Background(), "token"); err == nil { t.Fatal("unrecognized service account pod was accepted") } @@ -30,13 +30,13 @@ func TestAuthenticateWorkloadRejectsUnrecognizedServiceAccount(t *testing.T) { func TestAuthenticateWorkloadRejectsWrongAudienceOrDeletedPod(t *testing.T) { client := k8sfake.NewSimpleClientset() client.PrependReactor("create", "tokenreviews", tokenReviewReactor(runtimeWorkerServiceAccount, "games", "worker-0", "pod-uid", false)) - backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, nil, client) + backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, client) if _, err := backend.AuthenticateWorkload(context.Background(), "token"); err == nil { t.Fatal("unauthenticated token was accepted") } client = k8sfake.NewSimpleClientset() client.PrependReactor("create", "tokenreviews", tokenReviewReactor(runtimeWorkerServiceAccount, "games", "worker-0", "pod-uid", true)) - backend = NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, nil, client) + backend = NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, client) if _, err := backend.AuthenticateWorkload(context.Background(), "token"); err == nil { t.Fatal("deleted pod token was accepted") } @@ -50,7 +50,7 @@ func TestAuthenticateWorkloadRejectsCrossRuntimeLabel(t *testing.T) { Spec: corev1.PodSpec{ServiceAccountName: runtimeWorkerServiceAccount}, }) client.PrependReactor("create", "tokenreviews", tokenReviewReactor(runtimeWorkerServiceAccount, "games", "worker-0", "pod-uid", true)) - backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, nil, client) + backend := NewWithClient(Config{Namespace: "games", OperatorServiceAccount: "operator-system/operator"}, client) if _, err := backend.AuthenticateWorkload(context.Background(), "token"); err == nil { t.Fatal("worker without worker-pull label was accepted") } @@ -58,7 +58,7 @@ func TestAuthenticateWorkloadRejectsCrossRuntimeLabel(t *testing.T) { func TestEnsureRuntimeServiceAccountsCreatesBothAndIsIdempotent(t *testing.T) { client := k8sfake.NewSimpleClientset() - backend := NewWithClient(Config{Namespace: "games"}, nil, client) + backend := NewWithClient(Config{Namespace: "games"}, client) if err := backend.ensureRuntimeServiceAccounts(context.Background(), "games"); err != nil { t.Fatal(err) diff --git a/test/integration/example_test.go b/test/integration/example_test.go index e3224772..d4ad0977 100644 --- a/test/integration/example_test.go +++ b/test/integration/example_test.go @@ -26,8 +26,6 @@ type ServiceConfig struct { TestAddress string TestName string CommandStatus []string - UseLogSpy bool - LogSpy func(string, []byte) bool } func checkQueue(queueManager interface { @@ -51,11 +49,6 @@ func TestExamples(t *testing.T) { TestAddress: "localhost:25565", TestName: "Minecraft", CommandStatus: []string{"start", "install"}, - UseLogSpy: true, - LogSpy: func(stream string, sc []byte) bool { - println(string(sc)) - return strings.Contains(string(sc), `For help, type "help"`) - }, }, { ServiceName: "nginx", @@ -71,17 +64,6 @@ func TestExamples(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - logManager := mock_ports.NewMockLogManagerInterface(ctrl) - logDoneChan := make(chan struct{}, 1) - - logManager.EXPECT().AddLine(gomock.Any(), gomock.Any()).DoAndReturn(func(stream string, sc []byte) { - if config.UseLogSpy { - if config.LogSpy(stream, sc) { - logDoneChan <- struct{}{} - } - } - }).AnyTimes() - unixTime := time.Now().Unix() path := "./druid-cli-test/" @@ -148,10 +130,6 @@ func TestExamples(t *testing.T) { return } - if config.UseLogSpy { - <-logDoneChan - } - if config.TestAddress != "" { err = test_utils.ConnectionTest(config.TestAddress, true) if err != nil { diff --git a/test/mock/services.go b/test/mock/services.go index 0546777c..cd3883b0 100644 --- a/test/mock/services.go +++ b/test/mock/services.go @@ -11,10 +11,11 @@ package mock_ports import ( context "context" + io "io" reflect "reflect" time "time" - v2 "github.com/gofiber/fiber/v2" + fiber "github.com/gofiber/fiber/v2" domain "github.com/highcard-dev/daemon/internal/core/domain" ports "github.com/highcard-dev/daemon/internal/core/ports" v1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -47,7 +48,7 @@ func (m *MockAuthorizerServiceInterface) EXPECT() *MockAuthorizerServiceInterfac } // CheckHeader mocks base method. -func (m *MockAuthorizerServiceInterface) CheckHeader(r *v2.Ctx) (*ports.AuthContext, error) { +func (m *MockAuthorizerServiceInterface) CheckHeader(r *fiber.Ctx) (*ports.AuthContext, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CheckHeader", r) ret0, _ := ret[0].(*ports.AuthContext) @@ -185,56 +186,6 @@ func (mr *MockScrollServiceInterfaceMockRecorder) GetFile() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFile", reflect.TypeOf((*MockScrollServiceInterface)(nil).GetFile)) } -// MockLogManagerInterface is a mock of LogManagerInterface interface. -type MockLogManagerInterface struct { - ctrl *gomock.Controller - recorder *MockLogManagerInterfaceMockRecorder - isgomock struct{} -} - -// MockLogManagerInterfaceMockRecorder is the mock recorder for MockLogManagerInterface. -type MockLogManagerInterfaceMockRecorder struct { - mock *MockLogManagerInterface -} - -// NewMockLogManagerInterface creates a new mock instance. -func NewMockLogManagerInterface(ctrl *gomock.Controller) *MockLogManagerInterface { - mock := &MockLogManagerInterface{ctrl: ctrl} - mock.recorder = &MockLogManagerInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockLogManagerInterface) EXPECT() *MockLogManagerInterfaceMockRecorder { - return m.recorder -} - -// AddLine mocks base method. -func (m *MockLogManagerInterface) AddLine(stream string, sc []byte) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "AddLine", stream, sc) -} - -// AddLine indicates an expected call of AddLine. -func (mr *MockLogManagerInterfaceMockRecorder) AddLine(stream, sc any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLine", reflect.TypeOf((*MockLogManagerInterface)(nil).AddLine), stream, sc) -} - -// GetStreams mocks base method. -func (m *MockLogManagerInterface) GetStreams() map[string]*domain.Log { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStreams") - ret0, _ := ret[0].(map[string]*domain.Log) - return ret0 -} - -// GetStreams indicates an expected call of GetStreams. -func (mr *MockLogManagerInterfaceMockRecorder) GetStreams() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStreams", reflect.TypeOf((*MockLogManagerInterface)(nil).GetStreams)) -} - // MockRuntimeBackendInterface is a mock of RuntimeBackendInterface interface. type MockRuntimeBackendInterface struct { ctrl *gomock.Controller @@ -259,20 +210,6 @@ func (m *MockRuntimeBackendInterface) EXPECT() *MockRuntimeBackendInterfaceMockR return m.recorder } -// Attach mocks base method. -func (m *MockRuntimeBackendInterface) Attach(commandName, data string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attach", commandName, data) - ret0, _ := ret[0].(error) - return ret0 -} - -// Attach indicates an expected call of Attach. -func (mr *MockRuntimeBackendInterfaceMockRecorder) Attach(commandName, data any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attach", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).Attach), commandName, data) -} - // BackupRuntime mocks base method. func (m *MockRuntimeBackendInterface) BackupRuntime(ctx context.Context, root, artifact string, registryCredentials []domain.RegistryCredential) error { m.ctrl.T.Helper() @@ -317,18 +254,18 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) DeleteRuntime(root, purgeData } // ExpectedPorts mocks base method. -func (m *MockRuntimeBackendInterface) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimePortStatus, error) { +func (m *MockRuntimeBackendInterface) ExpectedPorts(root string, commands map[string]*domain.CommandInstructionSet, arg2 []domain.Port) ([]domain.RuntimePortStatus, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExpectedPorts", root, commands, ports) + ret := m.ctrl.Call(m, "ExpectedPorts", root, commands, arg2) ret0, _ := ret[0].([]domain.RuntimePortStatus) ret1, _ := ret[1].(error) return ret0, ret1 } // ExpectedPorts indicates an expected call of ExpectedPorts. -func (mr *MockRuntimeBackendInterfaceMockRecorder) ExpectedPorts(root, commands, ports any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) ExpectedPorts(root, commands, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpectedPorts", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).ExpectedPorts), root, commands, ports) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpectedPorts", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).ExpectedPorts), root, commands, arg2) } // Name mocks base method. @@ -345,6 +282,21 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) Name() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).Name)) } +// OpenConsole mocks base method. +func (m *MockRuntimeBackendInterface) OpenConsole(ctx context.Context, root, procedure string) (io.ReadWriteCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OpenConsole", ctx, root, procedure) + ret0, _ := ret[0].(io.ReadWriteCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OpenConsole indicates an expected call of OpenConsole. +func (mr *MockRuntimeBackendInterfaceMockRecorder) OpenConsole(ctx, root, procedure any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenConsole", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).OpenConsole), ctx, root, procedure) +} + // RootRef mocks base method. func (m *MockRuntimeBackendInterface) RootRef(id, namespace string) string { m.ctrl.T.Helper() @@ -360,18 +312,18 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) RootRef(id, namespace any) *g } // RoutingTargets mocks base method. -func (m *MockRuntimeBackendInterface) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, ports []domain.Port) ([]domain.RuntimeRoutingTarget, error) { +func (m *MockRuntimeBackendInterface) RoutingTargets(root string, commands map[string]*domain.CommandInstructionSet, arg2 []domain.Port) ([]domain.RuntimeRoutingTarget, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RoutingTargets", root, commands, ports) + ret := m.ctrl.Call(m, "RoutingTargets", root, commands, arg2) ret0, _ := ret[0].([]domain.RuntimeRoutingTarget) ret1, _ := ret[1].(error) return ret0, ret1 } // RoutingTargets indicates an expected call of RoutingTargets. -func (mr *MockRuntimeBackendInterfaceMockRecorder) RoutingTargets(root, commands, ports any) *gomock.Call { +func (mr *MockRuntimeBackendInterfaceMockRecorder) RoutingTargets(root, commands, arg2 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RoutingTargets", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).RoutingTargets), root, commands, ports) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RoutingTargets", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).RoutingTargets), root, commands, arg2) } // RunCommand mocks base method. @@ -404,11 +356,12 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) Signal(commandName, target, s } // SpawnPullWorker mocks base method. -func (m *MockRuntimeBackendInterface) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) error { +func (m *MockRuntimeBackendInterface) SpawnPullWorker(ctx context.Context, action ports.RuntimeWorkerAction) (<-chan error, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SpawnPullWorker", ctx, action) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(<-chan error) + ret1, _ := ret[1].(error) + return ret0, ret1 } // SpawnPullWorker indicates an expected call of SpawnPullWorker. @@ -417,6 +370,20 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) SpawnPullWorker(ctx, action a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SpawnPullWorker", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).SpawnPullWorker), ctx, action) } +// StopCommand mocks base method. +func (m *MockRuntimeBackendInterface) StopCommand(root, command string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StopCommand", root, command) + ret0, _ := ret[0].(error) + return ret0 +} + +// StopCommand indicates an expected call of StopCommand. +func (mr *MockRuntimeBackendInterfaceMockRecorder) StopCommand(root, command any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopCommand", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).StopCommand), root, command) +} + // StopRuntime mocks base method. func (m *MockRuntimeBackendInterface) StopRuntime(root string) error { m.ctrl.T.Helper() @@ -431,18 +398,43 @@ func (mr *MockRuntimeBackendInterfaceMockRecorder) StopRuntime(root any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopRuntime", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).StopRuntime), root) } -// StopCommand mocks base method. -func (m *MockRuntimeBackendInterface) StopCommand(root, command string) error { +// MockRuntimeBackendFactory is a mock of RuntimeBackendFactory interface. +type MockRuntimeBackendFactory struct { + ctrl *gomock.Controller + recorder *MockRuntimeBackendFactoryMockRecorder + isgomock struct{} +} + +// MockRuntimeBackendFactoryMockRecorder is the mock recorder for MockRuntimeBackendFactory. +type MockRuntimeBackendFactoryMockRecorder struct { + mock *MockRuntimeBackendFactory +} + +// NewMockRuntimeBackendFactory creates a new mock instance. +func NewMockRuntimeBackendFactory(ctrl *gomock.Controller) *MockRuntimeBackendFactory { + mock := &MockRuntimeBackendFactory{ctrl: ctrl} + mock.recorder = &MockRuntimeBackendFactoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRuntimeBackendFactory) EXPECT() *MockRuntimeBackendFactoryMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockRuntimeBackendFactory) Create(observer ports.ProcedureStatusObserver) (ports.RuntimeBackendInterface, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StopCommand", root, command) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "Create", observer) + ret0, _ := ret[0].(ports.RuntimeBackendInterface) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// StopCommand indicates an expected call of StopCommand. -func (mr *MockRuntimeBackendInterfaceMockRecorder) StopCommand(root, command any) *gomock.Call { +// Create indicates an expected call of Create. +func (mr *MockRuntimeBackendFactoryMockRecorder) Create(observer any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopCommand", reflect.TypeOf((*MockRuntimeBackendInterface)(nil).StopCommand), root, command) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRuntimeBackendFactory)(nil).Create), observer) } // MockRuntimeWorkerCallbackBackend is a mock of RuntimeWorkerCallbackBackend interface. @@ -661,121 +653,40 @@ func (mr *MockRuntimeScrollStoreMockRecorder) UpdateScroll(scroll any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateScroll", reflect.TypeOf((*MockRuntimeScrollStore)(nil).UpdateScroll), scroll) } -// MockBroadcastChannelInterface is a mock of BroadcastChannelInterface interface. -type MockBroadcastChannelInterface struct { - ctrl *gomock.Controller - recorder *MockBroadcastChannelInterfaceMockRecorder - isgomock struct{} -} - -// MockBroadcastChannelInterfaceMockRecorder is the mock recorder for MockBroadcastChannelInterface. -type MockBroadcastChannelInterfaceMockRecorder struct { - mock *MockBroadcastChannelInterface -} - -// NewMockBroadcastChannelInterface creates a new mock instance. -func NewMockBroadcastChannelInterface(ctrl *gomock.Controller) *MockBroadcastChannelInterface { - mock := &MockBroadcastChannelInterface{ctrl: ctrl} - mock.recorder = &MockBroadcastChannelInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBroadcastChannelInterface) EXPECT() *MockBroadcastChannelInterfaceMockRecorder { - return m.recorder -} - -// NewHub mocks base method. -func (m *MockBroadcastChannelInterface) NewHub() *domain.BroadcastChannel { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewHub") - ret0, _ := ret[0].(*domain.BroadcastChannel) - return ret0 -} - -// NewHub indicates an expected call of NewHub. -func (mr *MockBroadcastChannelInterfaceMockRecorder) NewHub() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewHub", reflect.TypeOf((*MockBroadcastChannelInterface)(nil).NewHub)) -} - -// Run mocks base method. -func (m *MockBroadcastChannelInterface) Run() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Run") -} - -// Run indicates an expected call of Run. -func (mr *MockBroadcastChannelInterfaceMockRecorder) Run() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockBroadcastChannelInterface)(nil).Run)) -} - -// MockConsoleManagerInterface is a mock of ConsoleManagerInterface interface. -type MockConsoleManagerInterface struct { +// MockProcedureStatusObserver is a mock of ProcedureStatusObserver interface. +type MockProcedureStatusObserver struct { ctrl *gomock.Controller - recorder *MockConsoleManagerInterfaceMockRecorder + recorder *MockProcedureStatusObserverMockRecorder isgomock struct{} } -// MockConsoleManagerInterfaceMockRecorder is the mock recorder for MockConsoleManagerInterface. -type MockConsoleManagerInterfaceMockRecorder struct { - mock *MockConsoleManagerInterface +// MockProcedureStatusObserverMockRecorder is the mock recorder for MockProcedureStatusObserver. +type MockProcedureStatusObserverMockRecorder struct { + mock *MockProcedureStatusObserver } -// NewMockConsoleManagerInterface creates a new mock instance. -func NewMockConsoleManagerInterface(ctrl *gomock.Controller) *MockConsoleManagerInterface { - mock := &MockConsoleManagerInterface{ctrl: ctrl} - mock.recorder = &MockConsoleManagerInterfaceMockRecorder{mock} +// NewMockProcedureStatusObserver creates a new mock instance. +func NewMockProcedureStatusObserver(ctrl *gomock.Controller) *MockProcedureStatusObserver { + mock := &MockProcedureStatusObserver{ctrl: ctrl} + mock.recorder = &MockProcedureStatusObserverMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConsoleManagerInterface) EXPECT() *MockConsoleManagerInterfaceMockRecorder { +func (m *MockProcedureStatusObserver) EXPECT() *MockProcedureStatusObserverMockRecorder { return m.recorder } -// AddConsoleWithChannel mocks base method. -func (m *MockConsoleManagerInterface) AddConsoleWithChannel(consoleId string, consoleType domain.ConsoleType, inputMode string, channel chan string) (*domain.Console, chan struct{}) { +// ObserveProcedureStatus mocks base method. +func (m *MockProcedureStatusObserver) ObserveProcedureStatus(update ports.ProcedureStatusUpdate) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddConsoleWithChannel", consoleId, consoleType, inputMode, channel) - ret0, _ := ret[0].(*domain.Console) - ret1, _ := ret[1].(chan struct{}) - return ret0, ret1 -} - -// AddConsoleWithChannel indicates an expected call of AddConsoleWithChannel. -func (mr *MockConsoleManagerInterfaceMockRecorder) AddConsoleWithChannel(consoleId, consoleType, inputMode, channel any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddConsoleWithChannel", reflect.TypeOf((*MockConsoleManagerInterface)(nil).AddConsoleWithChannel), consoleId, consoleType, inputMode, channel) -} - -// GetConsole mocks base method. -func (m *MockConsoleManagerInterface) GetConsole(consoleId string) *domain.Console { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConsole", consoleId) - ret0, _ := ret[0].(*domain.Console) - return ret0 -} - -// GetConsole indicates an expected call of GetConsole. -func (mr *MockConsoleManagerInterfaceMockRecorder) GetConsole(consoleId any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConsole", reflect.TypeOf((*MockConsoleManagerInterface)(nil).GetConsole), consoleId) -} - -// GetConsoles mocks base method. -func (m *MockConsoleManagerInterface) GetConsoles() map[string]*domain.Console { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConsoles") - ret0, _ := ret[0].(map[string]*domain.Console) - return ret0 + m.ctrl.Call(m, "ObserveProcedureStatus", update) } -// GetConsoles indicates an expected call of GetConsoles. -func (mr *MockConsoleManagerInterfaceMockRecorder) GetConsoles() *gomock.Call { +// ObserveProcedureStatus indicates an expected call of ObserveProcedureStatus. +func (mr *MockProcedureStatusObserverMockRecorder) ObserveProcedureStatus(update any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConsoles", reflect.TypeOf((*MockConsoleManagerInterface)(nil).GetConsoles)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObserveProcedureStatus", reflect.TypeOf((*MockProcedureStatusObserver)(nil).ObserveProcedureStatus), update) } // MockOciRegistryInterface is a mock of OciRegistryInterface interface.