fix(profiles): check that the agent stays up without NRestarts or --value

test/otel_collect/linux/profiles/agent_process.go

@@ -0,0 +1,71 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !windows
+
+package profiles
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/amazon-cloudwatch-agent-test/util/common"
+)
+
+const agentUnit = "amazon-cloudwatch-agent"
+
+// agentProcess is the agent unit's systemd state. restarts is empty when systemd does not report NRestarts.
+type agentProcess struct {
+	mainPID     string
+	restarts    string
+	activeState string
+}
+
+// runCommand runs a shell command; tests replace it to simulate older systemd versions.
+var runCommand = common.RunCommand
+
+// readAgentProcess reads the agent unit's MainPID and ActiveState, and NRestarts where systemd has it.
+//
+// The systemd on AL2 (219) has no NRestarts property and no `systemctl show --value` flag:
+// `systemctl show -p NRestarts` exits 1 there and prints only the properties it knows. So MainPID and
+// ActiveState, which every supported systemd reports, are read on their own, and an unchanged MainPID is
+// the restart proof everywhere. NRestarts is read separately and only adds a check where it exists.
+func readAgentProcess() (agentProcess, error) {
+	out, err := runCommand("systemctl show -p MainPID -p ActiveState " + agentUnit)
+	if err != nil {
+		return agentProcess{}, fmt.Errorf("systemctl show %s: %w", agentUnit, err)
+	}
+	p := parseAgentProcess(out)
+	if restarts, err := runCommand("systemctl show -p NRestarts " + agentUnit); err == nil {
+		p.restarts = parseAgentProcess(restarts).restarts
+	}
+	return p, nil
+}
+
+func parseAgentProcess(out string) agentProcess {
+	var p agentProcess
+	for _, line := range strings.Split(out, "\n") {
+		key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
+		if !ok {
+			continue
+		}
+		switch key {
+		case "MainPID":
+			p.mainPID = value
+		case "NRestarts":
+			p.restarts = value
+		case "ActiveState":
+			p.activeState = value
+		}
+	}
+	return p
+}
+
+// agentPID returns the agent's main PID, or "" when the unit has no running main process.
+func agentPID() string {
+	p, err := readAgentProcess()
+	if err != nil || p.mainPID == "" || p.mainPID == "0" {
+		return ""
+	}
+	return p.mainPID
+}

test/otel_collect/linux/profiles/attribute_cap_test.go

@@ -8,6 +8,7 @@ package profiles
 import (
 	"encoding/json"
 	"errors"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -38,6 +39,79 @@ func TestParseAgentProcess(t *testing.T) {
 	require.Equal(t, agentProcess{mainPID: "1234", restarts: "0", activeState: "active"}, p)
 }
 
+// fakeSystemctl behaves like `systemctl show` for a unit with MainPID 1234. With oldSystemd it acts like
+// AL2's systemd 219: no NRestarts property (exit 1, printing only the known properties) and no --value flag.
+func fakeSystemctl(t *testing.T, oldSystemd bool) {
+	t.Helper()
+	orig := runCommand
+	t.Cleanup(func() { runCommand = orig })
+	runCommand = func(cmd string) (string, error) {
+		if oldSystemd && strings.Contains(cmd, "--value") {
+			return "", errors.New("exit status 1: systemctl: unrecognized option '--value'")
+		}
+		var out []string
+		for _, prop := range []string{"MainPID=1234", "NRestarts=0", "ActiveState=active"} {
+			name, _, _ := strings.Cut(prop, "=")
+			if !strings.Contains(cmd, "-p "+name) || (oldSystemd && name == "NRestarts") {
+				continue
+			}
+			out = append(out, prop)
+		}
+		if oldSystemd && strings.Contains(cmd, "-p NRestarts") {
+			return strings.Join(out, "\n"), errors.New("exit status 1")
+		}
+		return strings.Join(out, "\n"), nil
+	}
+}
+
+func TestReadAgentProcess(t *testing.T) {
+	t.Run("systemd with NRestarts", func(t *testing.T) {
+		fakeSystemctl(t, false)
+		p, err := readAgentProcess()
+		require.NoError(t, err)
+		require.Equal(t, agentProcess{mainPID: "1234", restarts: "0", activeState: "active"}, p)
+	})
+	t.Run("AL2 systemd without NRestarts", func(t *testing.T) {
+		fakeSystemctl(t, true)
+		p, err := readAgentProcess()
+		require.NoError(t, err)
+		require.Equal(t, agentProcess{mainPID: "1234", activeState: "active"}, p)
+	})
+}
+
+func TestAgentPIDOnAL2(t *testing.T) {
+	fakeSystemctl(t, true)
+	require.Equal(t, "1234", agentPID(), "the main PID must be readable without systemctl --value")
+}
+
+func TestCheckAgentStaysUp(t *testing.T) {
+	up := agentProcess{mainPID: "1234", restarts: "0", activeState: "active"}
+	al2 := agentProcess{mainPID: "1234", activeState: "active"}
+	cases := []struct {
+		name          string
+		before, after agentProcess
+		wantOK        bool
+		wantReason    string
+	}{
+		{"unchanged with NRestarts", up, up, true, ""},
+		{"unchanged on AL2", al2, al2, true, ""},
+		{"restarted on AL2", al2, agentProcess{mainPID: "5678", activeState: "active"}, false, "main PID changed"},
+		{"restart counted", up, agentProcess{mainPID: "1234", restarts: "1", activeState: "active"}, false, "restart count changed"},
+		{"unit not active", up, agentProcess{mainPID: "0", restarts: "0", activeState: "failed"}, false, `"failed"`},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			r := (&ProfilesAttributeCapTestRunner{done: true, before: c.before, after: c.after}).checkAgentStaysUp()
+			if c.wantOK {
+				require.Equal(t, status.SUCCESSFUL, r.Status, "%v", r.Reason)
+				return
+			}
+			require.Equal(t, status.FAILED, r.Status)
+			require.ErrorContains(t, r.Reason, c.wantReason)
+		})
+	}
+}
+
 func TestCheckPartialSuccessSurfaced(t *testing.T) {
 	ok := checkPartialSuccessSurfaced([]string{
 		`W! {"caller":"otlphttpexporter/otlp.go:455","msg":"Partial success response","otelcol.component.id":"otlp_http/profiles","dropped_samples":1}`,

test/otel_collect/linux/profiles/other_signals.go

@@ -167,12 +167,3 @@ func postMetrics(payload []byte) error {
 	}
 	return nil
 }
-
-func agentPID() string {
-	out, err := common.RunCommand("systemctl show -p MainPID --value amazon-cloudwatch-agent")
-	pid := strings.TrimSpace(out)
-	if err != nil || pid == "" || pid == "0" {
-		return ""
-	}
-	return pid
-}

test/otel_collect/linux/profiles/profiles_attribute_cap_test.go

@@ -30,8 +30,6 @@ const (
 
 	capWarmupPushes     = 2
 	capSubsequentPushes = 3
-
-	agentUnit = "amazon-cloudwatch-agent"
 )
 
 type checkLevel string
@@ -46,12 +44,6 @@ type capCheck struct {
 	level checkLevel
 }
 
-type agentProcess struct {
-	mainPID     string
-	restarts    string
-	activeState string
-}
-
 type ProfilesAttributeCapTestRunner struct {
 	test_runner.BaseTestRunner
 	env *environment.MetaData
@@ -171,33 +163,6 @@ func buildAttributeCapPayload(serviceName, instanceID string, extra int) []byte
 	return buildResourceProfilesPayload(valid, oversized)
 }
 
-func readAgentProcess() (agentProcess, error) {
-	out, err := common.RunCommand("systemctl show -p MainPID -p NRestarts -p ActiveState " + agentUnit)
-	if err != nil {
-		return agentProcess{}, fmt.Errorf("systemctl show %s: %w", agentUnit, err)
-	}
-	return parseAgentProcess(out), nil
-}
-
-func parseAgentProcess(out string) agentProcess {
-	var p agentProcess
-	for _, line := range strings.Split(out, "\n") {
-		key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
-		if !ok {
-			continue
-		}
-		switch key {
-		case "MainPID":
-			p.mainPID = value
-		case "NRestarts":
-			p.restarts = value
-		case "ActiveState":
-			p.activeState = value
-		}
-	}
-	return p
-}
-
 func (t *ProfilesAttributeCapTestRunner) Validate() status.TestGroupResult {
 	checks := t.capChecks()
 	results := make([]status.TestResult, 0, len(checks))
@@ -255,7 +220,7 @@ func (t *ProfilesAttributeCapTestRunner) checkAgentStaysUp() status.TestResult {
 		r.Reason = fmt.Errorf("agent unit is %q after the oversized push", t.after.activeState)
 	case t.before.mainPID == "" || t.before.mainPID == "0" || t.before.mainPID != t.after.mainPID:
 		r.Reason = fmt.Errorf("agent main PID changed from %q to %q: it crashed or restarted", t.before.mainPID, t.after.mainPID)
-	case t.before.restarts != t.after.restarts:
+	case t.before.restarts != "" && t.after.restarts != "" && t.before.restarts != t.after.restarts:
 		r.Reason = fmt.Errorf("agent restart count changed from %s to %s", t.before.restarts, t.after.restarts)
 	default:
 		r.Status = status.SUCCESSFUL