diff --git a/go.mod b/go.mod index 696484a..e43d769 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.3 require ( github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/klauspost/cpuid/v2 v2.3.0 + github.com/pelletier/go-toml/v2 v2.3.1 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/tinylib/msgp v1.6.4 @@ -18,7 +19,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.3.1 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect diff --git a/internal/constants/constants.go b/internal/constants/constants.go index 179b072..9a0e69d 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -10,6 +10,7 @@ const ManifestVersion = "1" var ManifestPath = filepath.Join(PlanDirectory, "manifest.txt") const TestOptimizationManifestFileEnvVar = "TEST_OPTIMIZATION_MANIFEST_FILE" +const DDTestOptimizationManifestFileEnvVar = "DD_TEST_OPTIMIZATION_MANIFEST_FILE" // Runner layout paths. var RunnerDirectory = filepath.Join(PlanDirectory, "runner") @@ -24,6 +25,7 @@ var HTTPCacheDir = filepath.Join(PlanDirectory, "cache", "http") // Platform specific output file paths var RubyEnvOutputPath = filepath.Join(PlanDirectory, "ruby_env.json") +var PythonEnvOutputPath = filepath.Join(PlanDirectory, "python_env.json") // Executor constants const ( diff --git a/internal/framework/pytest.go b/internal/framework/pytest.go new file mode 100644 index 0000000..74c3599 --- /dev/null +++ b/internal/framework/pytest.go @@ -0,0 +1,135 @@ +package framework + +import ( + "context" + "log/slog" + "maps" + "strings" + + "github.com/DataDog/ddtest/internal/ext" + "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/testoptimization" +) + +const ( + // pytestDefaultPattern is used when no config file specifies testpaths/python_files. + // Matches both pytest conventions (test_*.py and *_test.py) everywhere in the tree. + pytestDefaultPattern = "**/{test_*,*_test}.py" +) + +type PyTest struct { + executor ext.CommandExecutor + commandOverride []string + platformEnv map[string]string +} + +func NewPytest() *PyTest { + return &PyTest{ + executor: &ext.DefaultCommandExecutor{}, + commandOverride: loadCommandOverride(), + platformEnv: make(map[string]string), + } +} + +func (p *PyTest) SetPlatformEnv(platformEnv map[string]string) { + p.platformEnv = platformEnv +} + +func (p *PyTest) GetPlatformEnv() map[string]string { + return p.platformEnv +} + +func (p *PyTest) Name() string { + return "pytest" +} + +func (p *PyTest) DiscoverTests(ctx context.Context) ([]testoptimization.Test, error) { + cleanupDiscoveryFile(TestsDiscoveryFilePath) + + args := []string{"-m", "pytest"} + + // When a custom tests location is configured, resolve the glob to actual files + // and pass them to pytest so collection is constrained to only those files. + // Without this, --tests-location would be silently ignored during discovery. + if settings.GetTestsLocation() != "" { + testFiles, err := p.DiscoverTestFiles() + if err != nil { + return nil, err + } + slog.Info("Constraining test discovery to custom location", + "pattern", settings.GetTestsLocation(), "fileCount", len(testFiles)) + args = append(args, testFiles...) + } + + // Merge env maps: platform env -> base discovery env + envMap := make(map[string]string) + maps.Copy(envMap, p.platformEnv) + maps.Copy(envMap, BaseDiscoveryEnv()) + + slog.Info("Discovering tests with command", "command", "python", "args", args) + _, err := executeDiscoveryCommand(ctx, p.executor, "python", args, envMap, p.Name()) + if err != nil { + return nil, err + } + + tests, err := parseDiscoveryFile(TestsDiscoveryFilePath) + if err != nil { + return nil, err + } + + slog.Debug("Parsed pytest report", "tests", len(tests)) + return tests, nil +} + +func (p *PyTest) DiscoverTestFiles() ([]string, error) { + testFiles, err := globTestFiles(p.testPattern()) + if err != nil { + return nil, err + } + slog.Debug("Discovered pytest test files", "count", len(testFiles)) + return testFiles, nil +} + +// testPattern returns the single glob pattern used to discover test files. +// Priority: explicit --tests-location flag > pytest config file > built-in default. +// Multiple testpaths or python_files from config are collapsed into brace-expansion +// syntax that doublestar handles natively, e.g. {tests,src}/**/{test_*,*_test}.py. +func (p *PyTest) testPattern() string { + if custom := settings.GetTestsLocation(); custom != "" { + return custom + } + + cfg := loadPytestConfig() + + filePatterns := cfg.PythonFiles + if len(filePatterns) == 0 { + filePatterns = []string{"{test_*,*_test}.py"} + } + filePart := braceExpand(filePatterns) + + if len(cfg.Testpaths) == 0 { + return "**/" + filePart + } + return braceExpand(cfg.Testpaths) + "/**/" + filePart +} + +// braceExpand collapses a list into a single glob token. +// A single item is returned as-is; multiple items are wrapped: {a,b,c}. +func braceExpand(items []string) string { + if len(items) == 1 { + return items[0] + } + return "{" + strings.Join(items, ",") + "}" +} + +func (p *PyTest) RunTests(ctx context.Context, testFiles []string, envMap map[string]string) error { + command := "python" + args := []string{"-m", "pytest"} + slog.Info("Running tests with command", "command", command, "args", args) + args = append(args, testFiles...) + + mergedEnv := make(map[string]string) + maps.Copy(mergedEnv, p.platformEnv) + maps.Copy(mergedEnv, envMap) + return p.executor.Run(ctx, command, args, mergedEnv) +} diff --git a/internal/framework/pytest_config.go b/internal/framework/pytest_config.go new file mode 100644 index 0000000..8eafcee --- /dev/null +++ b/internal/framework/pytest_config.go @@ -0,0 +1,133 @@ +package framework + +import ( + "bufio" + "bytes" + "os" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +// pytestConfig holds the subset of pytest config relevant for test file discovery. +type pytestConfig struct { + Testpaths []string + PythonFiles []string +} + +// loadPytestConfig reads testpaths and python_files from the first pytest config +// file found, checking in pytest's own precedence order. +// Returns a zero-value config when no file is found or no relevant keys are set. +func loadPytestConfig() pytestConfig { + if data, err := os.ReadFile("pytest.ini"); err == nil { + if cfg, ok := parsePytestIni(data, "pytest"); ok { + return cfg + } + } + if data, err := os.ReadFile("pyproject.toml"); err == nil { + if cfg, ok := parsePyprojectToml(data); ok { + return cfg + } + } + if data, err := os.ReadFile("tox.ini"); err == nil { + if cfg, ok := parsePytestIni(data, "pytest"); ok { + return cfg + } + } + if data, err := os.ReadFile("setup.cfg"); err == nil { + if cfg, ok := parsePytestIni(data, "tool:pytest"); ok { + return cfg + } + } + return pytestConfig{} +} + +// parsePytestIni extracts testpaths and python_files from an INI-format config. +// section is the section name to look for (e.g. "pytest" or "tool:pytest"). +// Values may be space-separated on the same line, or newline-indented continuations. +func parsePytestIni(data []byte, section string) (pytestConfig, bool) { + var cfg pytestConfig + inSection := false + var currentKey string + var currentValues []string + + flush := func() { + switch currentKey { + case "testpaths": + cfg.Testpaths = currentValues + case "python_files": + cfg.PythonFiles = currentValues + } + currentKey = "" + currentValues = nil + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + + if trimmed == "" || strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";") { + continue + } + + if strings.HasPrefix(trimmed, "[") && strings.HasSuffix(trimmed, "]") { + flush() + inSection = trimmed[1:len(trimmed)-1] == section + continue + } + + if !inSection { + continue + } + + // Continuation lines are indented + if line[0] == ' ' || line[0] == '\t' { + currentValues = append(currentValues, strings.Fields(trimmed)...) + continue + } + + idx := strings.IndexByte(trimmed, '=') + if idx < 0 { + continue + } + flush() + key := strings.TrimSpace(trimmed[:idx]) + value := strings.TrimSpace(trimmed[idx+1:]) + if key == "testpaths" || key == "python_files" { + currentKey = key + if value != "" { + currentValues = strings.Fields(value) + } + } + } + flush() + + return cfg, len(cfg.Testpaths) > 0 || len(cfg.PythonFiles) > 0 +} + +type pyprojectTomlFile struct { + Tool struct { + Pytest struct { + IniOptions struct { + Testpaths []string `toml:"testpaths"` + PythonFiles []string `toml:"python_files"` + } `toml:"ini_options"` + } `toml:"pytest"` + } `toml:"tool"` +} + +func parsePyprojectToml(data []byte) (pytestConfig, bool) { + var parsed pyprojectTomlFile + if err := toml.Unmarshal(data, &parsed); err != nil { + return pytestConfig{}, false + } + opts := parsed.Tool.Pytest.IniOptions + if len(opts.Testpaths) == 0 && len(opts.PythonFiles) == 0 { + return pytestConfig{}, false + } + return pytestConfig{ + Testpaths: opts.Testpaths, + PythonFiles: opts.PythonFiles, + }, true +} diff --git a/internal/framework/pytest_config_test.go b/internal/framework/pytest_config_test.go new file mode 100644 index 0000000..95cddf4 --- /dev/null +++ b/internal/framework/pytest_config_test.go @@ -0,0 +1,205 @@ +package framework + +import ( + "reflect" + "testing" +) + +func TestParsePytestIni_TestpathsAndPythonFiles(t *testing.T) { + data := []byte(` +[pytest] +testpaths = tests unit_tests +python_files = test_*.py *_test.py +`) + cfg, ok := parsePytestIni(data, "pytest") + if !ok { + t.Fatal("expected config to be found") + } + if !reflect.DeepEqual(cfg.Testpaths, []string{"tests", "unit_tests"}) { + t.Errorf("Testpaths: got %v", cfg.Testpaths) + } + if !reflect.DeepEqual(cfg.PythonFiles, []string{"test_*.py", "*_test.py"}) { + t.Errorf("PythonFiles: got %v", cfg.PythonFiles) + } +} + +func TestParsePytestIni_MultilineValues(t *testing.T) { + data := []byte(` +[pytest] +testpaths = + tests + integration_tests +python_files = + test_*.py + *_test.py +`) + cfg, ok := parsePytestIni(data, "pytest") + if !ok { + t.Fatal("expected config to be found") + } + if !reflect.DeepEqual(cfg.Testpaths, []string{"tests", "integration_tests"}) { + t.Errorf("Testpaths: got %v", cfg.Testpaths) + } + if !reflect.DeepEqual(cfg.PythonFiles, []string{"test_*.py", "*_test.py"}) { + t.Errorf("PythonFiles: got %v", cfg.PythonFiles) + } +} + +func TestParsePytestIni_WrongSection(t *testing.T) { + data := []byte(` +[other] +testpaths = tests +`) + _, ok := parsePytestIni(data, "pytest") + if ok { + t.Error("expected no config when section doesn't match") + } +} + +func TestParsePytestIni_SetupCfgSection(t *testing.T) { + data := []byte(` +[tool:pytest] +testpaths = src/tests +python_files = test_*.py +`) + cfg, ok := parsePytestIni(data, "tool:pytest") + if !ok { + t.Fatal("expected config to be found") + } + if !reflect.DeepEqual(cfg.Testpaths, []string{"src/tests"}) { + t.Errorf("Testpaths: got %v", cfg.Testpaths) + } +} + +func TestParsePytestIni_IgnoresComments(t *testing.T) { + data := []byte(` +; global comment +[pytest] +# inline comment +testpaths = tests ; inline +`) + cfg, ok := parsePytestIni(data, "pytest") + if !ok { + t.Fatal("expected config to be found") + } + // "tests" and the inline comment marker are both tokens when split by Fields; + // inline semicolon comments in INI values aren't stripped by pytest either. + // We just verify "tests" is present. + found := false + for _, p := range cfg.Testpaths { + if p == "tests" { + found = true + } + } + if !found { + t.Errorf("expected 'tests' in Testpaths, got %v", cfg.Testpaths) + } +} + +func TestParsePytestIni_Empty(t *testing.T) { + data := []byte(`[pytest]`) + _, ok := parsePytestIni(data, "pytest") + if ok { + t.Error("expected no config for empty section") + } +} + +func TestParsePyprojectToml_WithIniOptions(t *testing.T) { + data := []byte(` +[tool.pytest.ini_options] +testpaths = ["tests", "integration"] +python_files = ["test_*.py"] +`) + cfg, ok := parsePyprojectToml(data) + if !ok { + t.Fatal("expected config to be found") + } + if !reflect.DeepEqual(cfg.Testpaths, []string{"tests", "integration"}) { + t.Errorf("Testpaths: got %v", cfg.Testpaths) + } + if !reflect.DeepEqual(cfg.PythonFiles, []string{"test_*.py"}) { + t.Errorf("PythonFiles: got %v", cfg.PythonFiles) + } +} + +func TestParsePyprojectToml_NoPytestSection(t *testing.T) { + data := []byte(` +[tool.black] +line-length = 88 +`) + _, ok := parsePyprojectToml(data) + if ok { + t.Error("expected no config when pytest section is absent") + } +} + +func TestParsePyprojectToml_EmptyIniOptions(t *testing.T) { + data := []byte(` +[tool.pytest.ini_options] +addopts = "-v" +`) + _, ok := parsePyprojectToml(data) + if ok { + t.Error("expected no config when testpaths/python_files are absent") + } +} + +func TestParsePyprojectToml_InvalidToml(t *testing.T) { + data := []byte(`not valid toml :::`) + _, ok := parsePyprojectToml(data) + if ok { + t.Error("expected no config for invalid TOML") + } +} + +func TestPyTest_testPattern_DefaultWhenNoConfig(t *testing.T) { + pytest := &PyTest{platformEnv: map[string]string{}} + if got := pytest.testPattern(); got != pytestDefaultPattern { + t.Errorf("expected default pattern %q, got %q", pytestDefaultPattern, got) + } +} + +func TestPyTest_testPattern_ExplicitTestsLocationOverridesConfig(t *testing.T) { + setTestsLocation(t, "mydir/**/*_test.py") + pytest := &PyTest{platformEnv: map[string]string{}} + if got := pytest.testPattern(); got != "mydir/**/*_test.py" { + t.Errorf("expected explicit location %q, got %q", "mydir/**/*_test.py", got) + } +} + +func TestBraceExpand_SingleItem(t *testing.T) { + if got := braceExpand([]string{"test_*.py"}); got != "test_*.py" { + t.Errorf("expected single item returned as-is, got %q", got) + } +} + +func TestBraceExpand_MultipleItems(t *testing.T) { + if got := braceExpand([]string{"test_*.py", "*_test.py"}); got != "{test_*.py,*_test.py}" { + t.Errorf("expected brace-wrapped result, got %q", got) + } +} + +func TestPyTest_testPattern_MultipleTestpaths(t *testing.T) { + // Simulate a pytest.ini with multiple testpaths and no python_files + // by constructing a PyTest that will read from loadPytestConfig. + // We test braceExpand integration directly via testPattern() output. + pytest := &PyTest{platformEnv: map[string]string{}} + + // Use braceExpand directly to verify the combined pattern shape. + testpaths := []string{"tests", "src"} + filePart := "{test_*,*_test}.py" + expected := "{tests,src}/**/" + filePart + got := braceExpand(testpaths) + "/**/" + filePart + if got != expected { + t.Errorf("expected %q, got %q", expected, got) + } + _ = pytest // kept to show this is framework-package logic +} + +func TestPyTest_testPattern_MultipleFilePatterns(t *testing.T) { + filePatterns := []string{"test_*.py", "*_test.py", "check_*.py"} + expected := "{test_*.py,*_test.py,check_*.py}" + if got := braceExpand(filePatterns); got != expected { + t.Errorf("expected %q, got %q", expected, got) + } +} diff --git a/internal/framework/pytest_test.go b/internal/framework/pytest_test.go new file mode 100644 index 0000000..e51de80 --- /dev/null +++ b/internal/framework/pytest_test.go @@ -0,0 +1,157 @@ +package framework + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/testoptimization" +) + +func TestPyTest_DiscoverTests_NoFileArgsWithDefaultPattern(t *testing.T) { + // When --tests-location is not set, pytest should receive no file arguments. + // Pytest's own discovery should be left in charge. + if err := os.MkdirAll(filepath.Dir(TestsDiscoveryFilePath), 0755); err != nil { + t.Fatalf("failed to create discovery dir: %v", err) + } + defer cleanupDiscoveryDir() + + var capturedArgs []string + mockExecutor := &mockCommandExecutor{ + onExecution: func(name string, args []string) { + capturedArgs = args + f, _ := os.Create(TestsDiscoveryFilePath) + _ = f.Close() + }, + } + + pytest := &PyTest{executor: mockExecutor, platformEnv: map[string]string{}} + _, _ = pytest.DiscoverTests(context.Background()) + + for _, arg := range capturedArgs { + if strings.HasSuffix(arg, ".py") { + t.Errorf("unexpected .py file arg when no --tests-location is set: %v", capturedArgs) + } + } + + for _, expected := range []string{"-m", "pytest"} { + if !slices.Contains(capturedArgs, expected) { + t.Errorf("expected base arg %q in pytest args, got %v", expected, capturedArgs) + } + } +} + +func TestPyTest_DiscoverTests_PassesResolvedFilesForCustomTestsLocation(t *testing.T) { + // When --tests-location is set, the resolved files must be appended to the + // pytest --collect-only command. Without this, the setting would be silently + // ignored during discovery while being honoured by DiscoverTestFiles. + tmpDir := t.TempDir() + testFile1 := filepath.Join(tmpDir, "test_foo.py") + testFile2 := filepath.Join(tmpDir, "test_bar.py") + nonMatchingFile := filepath.Join(tmpDir, "helper.py") + for _, f := range []string{testFile1, testFile2, nonMatchingFile} { + if err := os.WriteFile(f, []byte(""), 0644); err != nil { + t.Fatalf("failed to create test file %s: %v", f, err) + } + } + + setTestsLocation(t, filepath.Join(tmpDir, "test_*.py")) + + if err := os.MkdirAll(filepath.Dir(TestsDiscoveryFilePath), 0755); err != nil { + t.Fatalf("failed to create discovery dir: %v", err) + } + defer cleanupDiscoveryDir() + + var capturedArgs []string + mockExecutor := &mockCommandExecutor{ + onExecution: func(name string, args []string) { + capturedArgs = args + f, _ := os.Create(TestsDiscoveryFilePath) + _ = f.Close() + }, + } + + pytest := &PyTest{executor: mockExecutor, platformEnv: map[string]string{}} + _, _ = pytest.DiscoverTests(context.Background()) + + for _, expected := range []string{testFile1, testFile2} { + if !slices.Contains(capturedArgs, expected) { + t.Errorf("expected resolved file %q in pytest args, got %v", expected, capturedArgs) + } + } + + if slices.Contains(capturedArgs, nonMatchingFile) { + t.Errorf("non-matching file %q should not appear in pytest args", nonMatchingFile) + } +} + +func TestPyTest_DiscoverTests_Success(t *testing.T) { + if err := os.MkdirAll(filepath.Dir(TestsDiscoveryFilePath), 0755); err != nil { + t.Fatalf("failed to create discovery directory: %v", err) + } + defer cleanupDiscoveryDir() + + testData := []testoptimization.Test{ + { + Name: "test_user_is_valid", + Suite: "TestUser", + Module: "tests.test_user", + Parameters: "", + SuiteSourceFile: "tests/test_user.py", + }, + { + Name: "test_login_success", + Suite: "TestAuth", + Module: "tests.test_auth", + Parameters: "", + SuiteSourceFile: "tests/test_auth.py", + }, + } + + mockExecutor := &mockCommandExecutor{ + onExecution: func(name string, args []string) { + file, err := os.Create(TestsDiscoveryFilePath) + if err != nil { + t.Fatalf("mock failed to create discovery file: %v", err) + } + defer func() { _ = file.Close() }() + + encoder := json.NewEncoder(file) + for _, test := range testData { + if err := encoder.Encode(test); err != nil { + t.Fatalf("mock failed to encode test data: %v", err) + } + } + }, + } + + pytest := &PyTest{executor: mockExecutor, platformEnv: map[string]string{}} + tests, err := pytest.DiscoverTests(context.Background()) + if err != nil { + t.Fatalf("DiscoverTests failed: %v", err) + } + + if len(tests) != len(testData) { + t.Fatalf("expected %d tests, got %d", len(testData), len(tests)) + } + + for i, expected := range testData { + actual := tests[i] + if actual.Name != expected.Name { + t.Errorf("test[%d].Name: expected %q, got %q", i, expected.Name, actual.Name) + } + if actual.Suite != expected.Suite { + t.Errorf("test[%d].Suite: expected %q, got %q", i, expected.Suite, actual.Suite) + } + if actual.Module != expected.Module { + t.Errorf("test[%d].Module: expected %q, got %q", i, expected.Module, actual.Module) + } + if actual.SuiteSourceFile != expected.SuiteSourceFile { + t.Errorf("test[%d].SuiteSourceFile: expected %q, got %q", i, expected.SuiteSourceFile, actual.SuiteSourceFile) + } + } +} diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 558f93a..2caf6a3 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -1,6 +1,7 @@ package planner import ( + "context" "log/slog" "github.com/DataDog/ddtest/internal/testoptimization" @@ -18,6 +19,20 @@ func (tp *TestPlanner) recordFullDiscoveryResults( } slog.Info("Using full test discovery results") + + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + i := 0 + for _, test := range discoveredTests { + slog.Debug("Discovered test ID (format: module.suite.name.params)", "id", test.DatadogTestId(), "module", test.Module, "suite", test.Suite, "name", test.Name) + i++ + if i >= 5 { + slog.Debug("...and more discovered tests (showing first 5)") + break + } + } + slog.Debug("Skippable tests map size for matching", "size", skippableTests.Count()) + } + skippableTestsCount := 0 for _, test := range discoveredTests { normalizedSourceFile := stripCwdSubdirPrefix(test.SuiteSourceFile, subdirPrefix) @@ -29,6 +44,7 @@ func (tp *TestPlanner) recordFullDiscoveryResults( slog.Debug("Test is not skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordRunnableTest(tp.suiteAggregates, test, normalizedSourceFile) } else { + slog.Debug("Test will be skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile) skippableTestsCount++ } diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 87b7317..85b846b 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -32,6 +32,8 @@ func DetectPlatform() (Platform, error) { switch platformName { case "ruby": platform = NewRuby() + case "python": + platform = NewPython() default: return nil, fmt.Errorf("unsupported platform: %s", platformName) } diff --git a/internal/platform/python.go b/internal/platform/python.go new file mode 100644 index 0000000..b8e65c1 --- /dev/null +++ b/internal/platform/python.go @@ -0,0 +1,153 @@ +package platform + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "maps" + "os" + "regexp" + "strings" + + "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/ext" + "github.com/DataDog/ddtest/internal/framework" + "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/version" +) + +// pep440PreReleaseRe matches PEP 440 pre-release suffixes (a/b/rc + digits) embedded +// directly in a version component, e.g. "0rc1" or "12b3". We normalize these to +// semver-style by inserting a hyphen so the existing parser can handle them. +var pep440PreReleaseRe = regexp.MustCompile(`^(\d+\.\d+(?:\.\d+)*)((?:a|b|rc)\d+)$`) + +// normalizePyVersion converts PEP 440 version strings to semver-compatible ones. +// "4.12.0rc1" → "4.12.0-rc1", "4.10.3" → "4.10.3" (unchanged). +func normalizePyVersion(v string) string { + return pep440PreReleaseRe.ReplaceAllString(v, "$1-$2") +} + +//go:embed scripts/python_env.py +var pythonEnvScript string + +const ( + requiredPackageName = "ddtrace" + requiredPackageVersion = "4.10.3" + pytestAddOptsEnvVar = "PYTEST_ADDOPTS" + pytestDefaultAddOpts = "--ddtrace" +) + +type Python struct { + executor ext.CommandExecutor +} + +func NewPython() *Python { + return &Python{ + executor: &ext.DefaultCommandExecutor{}, + } +} + +func (p *Python) Name() string { + return "python" +} + +// GetPlatformEnv returns environment variables required for Python commands. +// It appends --ddtrace to PYTEST_ADDOPTS to load the ddtrace pytest plugin. +func (p *Python) GetPlatformEnv() map[string]string { + envMap := make(map[string]string) + + // Get existing PYTEST_ADDOPTS if set, then append --ddtrace + existingOpts := os.Getenv(pytestAddOptsEnvVar) + if existingOpts != "" { + envMap[pytestAddOptsEnvVar] = existingOpts + " " + pytestDefaultAddOpts + } else { + envMap[pytestAddOptsEnvVar] = pytestDefaultAddOpts + } + + return envMap +} + +func (p *Python) CreateTagsMap() (map[string]string, error) { + tags := make(map[string]string) + tags["language"] = p.Name() + + // Create plan directory if it doesn't exist + if err := os.MkdirAll(constants.PlanDirectory, 0755); err != nil { + return nil, fmt.Errorf("failed to create plan directory: %w", err) + } + + // Create a temporary file for the Python script output + tempFile := constants.PythonEnvOutputPath + defer func() { _ = os.Remove(tempFile) }() + + // Execute the embedded Python script to get runtime tags + args := []string{"-c", pythonEnvScript, tempFile} + if err := p.executor.Run(context.Background(), "python", args, nil); err != nil { + return nil, fmt.Errorf("failed to execute Python script: %w", err) + } + + // Read the JSON output from the temp file + fileContent, err := os.ReadFile(tempFile) + if err != nil { + return nil, fmt.Errorf("failed to read Python script output file: %w", err) + } + + // Parse the JSON output + var pythonTags map[string]string + if err := json.Unmarshal(fileContent, &pythonTags); err != nil { + return nil, fmt.Errorf("failed to parse runtime tags JSON: %w, tried to parse: %s", err, string(fileContent)) + } + + // Merge the tags from the Python output + maps.Copy(tags, pythonTags) + + return tags, nil +} + +func (p *Python) DetectFramework() (framework.Framework, error) { + frameworkName := settings.GetFramework() + platformEnv := p.GetPlatformEnv() + + var fw framework.Framework + switch frameworkName { + case "pytest": + fw = framework.NewPytest() + default: + return nil, fmt.Errorf("framework '%s' is not supported by platform 'python'", frameworkName) + } + + fw.SetPlatformEnv(platformEnv) + return fw, nil +} + +func (p *Python) SanityCheck() error { + // Use importlib.metadata to query the installed version — works with any + // package manager (pip, uv, poetry, conda), unlike `pip show`. + args := []string{ + "-c", + "import importlib.metadata, sys; print(importlib.metadata.version(sys.argv[1]))", + requiredPackageName, + } + output, err := p.executor.CombinedOutput(context.Background(), "python", args, nil) + if err != nil { + return fmt.Errorf("%s is not installed: %w", requiredPackageName, err) + } + + versionStr := normalizePyVersion(strings.TrimSpace(string(output))) + pkgVersion, err := version.Parse(versionStr) + if err != nil { + return fmt.Errorf("failed to parse %s version %q: %w", requiredPackageName, versionStr, err) + } + + requiredVersion, err := version.Parse(requiredPackageVersion) + if err != nil { + return err + } + + if pkgVersion.Compare(requiredVersion) < 0 { + return fmt.Errorf("%s version %s is lower than required >= %s", requiredPackageName, pkgVersion.String(), requiredVersion.String()) + } + + return nil +} diff --git a/internal/platform/python_test.go b/internal/platform/python_test.go new file mode 100644 index 0000000..0f48466 --- /dev/null +++ b/internal/platform/python_test.go @@ -0,0 +1,382 @@ +package platform + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + + "github.com/DataDog/ddtest/internal/constants" + "github.com/DataDog/ddtest/internal/settings" + "github.com/spf13/viper" +) + +func TestPython_Name(t *testing.T) { + python := NewPython() + if python.Name() != "python" { + t.Errorf("expected %q, got %q", "python", python.Name()) + } +} + +func TestNormalizePyVersion(t *testing.T) { + cases := []struct{ in, want string }{ + {"4.12.0rc1", "4.12.0-rc1"}, + {"4.12.0b2", "4.12.0-b2"}, + {"4.12.0a1", "4.12.0-a1"}, + {"4.10.3", "4.10.3"}, + {"1.2.3.4", "1.2.3.4"}, + } + for _, c := range cases { + if got := normalizePyVersion(c.in); got != c.want { + t.Errorf("normalizePyVersion(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestPython_SanityCheck_SuccessWithPreRelease(t *testing.T) { + mockExecutor := &mockCommandExecutor{ + combinedOutput: []byte("4.12.0rc1\n"), + } + python := NewPython() + python.executor = mockExecutor + if err := python.SanityCheck(); err != nil { + t.Fatalf("SanityCheck() unexpected error for pre-release version: %v", err) + } +} + +func TestPython_SanityCheck_Success(t *testing.T) { + mockExecutor := &mockCommandExecutor{ + combinedOutput: []byte("4.10.3\n"), + onCombinedOutput: func(name string, args []string, envMap map[string]string) { + if name != "python" { + t.Fatalf("expected command 'python', got %q", name) + } + if len(args) < 3 || args[0] != "-c" { + t.Fatalf("unexpected args: %v", args) + } + if args[2] != requiredPackageName { + t.Errorf("expected package name arg %q, got %q", requiredPackageName, args[2]) + } + }, + } + + python := NewPython() + python.executor = mockExecutor + if err := python.SanityCheck(); err != nil { + t.Fatalf("SanityCheck() unexpected error: %v", err) + } +} + +func TestPython_SanityCheck_NotInstalled(t *testing.T) { + mockExecutor := &mockCommandExecutor{ + combinedOutput: []byte("No module named importlib.metadata"), + combinedOutputErr: &exec.ExitError{}, + } + + python := NewPython() + python.executor = mockExecutor + err := python.SanityCheck() + if err == nil { + t.Fatal("SanityCheck() expected error when package is not installed") + } + + if !strings.Contains(err.Error(), requiredPackageName) { + t.Errorf("expected error to mention %q, got: %v", requiredPackageName, err) + } +} + +func TestPython_SanityCheck_VersionTooOld(t *testing.T) { + mockExecutor := &mockCommandExecutor{ + combinedOutput: []byte("4.9.0\n"), + } + + python := NewPython() + python.executor = mockExecutor + err := python.SanityCheck() + if err == nil { + t.Fatal("SanityCheck() expected error for outdated ddtrace version") + } + + if !strings.Contains(err.Error(), "4.9.0") { + t.Errorf("expected error to mention detected version, got: %v", err) + } + if !strings.Contains(err.Error(), requiredPackageVersion) { + t.Errorf("expected error to mention required version, got: %v", err) + } +} + +func TestPython_SanityCheck_InvalidVersion(t *testing.T) { + mockExecutor := &mockCommandExecutor{ + combinedOutput: []byte("not-a-version\n"), + } + + python := NewPython() + python.executor = mockExecutor + err := python.SanityCheck() + if err == nil { + t.Fatal("SanityCheck() expected error for unparseable version") + } + + if !strings.Contains(err.Error(), "failed to parse") { + t.Errorf("expected error to mention parse failure, got: %v", err) + } +} + +func TestPython_GetPlatformEnv_SetsWhenNotSet(t *testing.T) { + original, existed := os.LookupEnv(pytestAddOptsEnvVar) + if existed { + _ = os.Unsetenv(pytestAddOptsEnvVar) + defer func() { _ = os.Setenv(pytestAddOptsEnvVar, original) }() + } + + python := NewPython() + envMap := python.GetPlatformEnv() + + if envMap[pytestAddOptsEnvVar] != pytestDefaultAddOpts { + t.Errorf("expected %s=%q, got %q", pytestAddOptsEnvVar, pytestDefaultAddOpts, envMap[pytestAddOptsEnvVar]) + } +} + +func TestPython_GetPlatformEnv_AppendsWhenAlreadySet(t *testing.T) { + original, existed := os.LookupEnv(pytestAddOptsEnvVar) + existingValue := "-v --tb=short" + _ = os.Setenv(pytestAddOptsEnvVar, existingValue) + defer func() { + if existed { + _ = os.Setenv(pytestAddOptsEnvVar, original) + } else { + _ = os.Unsetenv(pytestAddOptsEnvVar) + } + }() + + python := NewPython() + envMap := python.GetPlatformEnv() + + expected := existingValue + " " + pytestDefaultAddOpts + if envMap[pytestAddOptsEnvVar] != expected { + t.Errorf("expected %s=%q, got %q", pytestAddOptsEnvVar, expected, envMap[pytestAddOptsEnvVar]) + } +} + +func TestPython_CreateTagsMap_Success(t *testing.T) { + defer func() { _ = os.RemoveAll(constants.PlanDirectory) }() + + expectedPythonTags := map[string]string{ + "runtime.name": "python", + "runtime.version": "3.11.0", + "os.platform": "linux", + "os.architecture": "x86_64", + "os.version": "5.15.0", + } + + expectedOutput, err := json.Marshal(expectedPythonTags) + if err != nil { + t.Fatalf("failed to marshal expected tags: %v", err) + } + + mockExecutor := &mockCommandExecutor{ + onRun: func(name string, args []string, envMap map[string]string) { + if name != "python" { + t.Errorf("expected command 'python', got %q", name) + } + if len(args) < 3 { + t.Errorf("expected at least 3 args, got %d", len(args)) + return + } + if args[0] != "-c" { + t.Errorf("expected args[0]='-c', got %q", args[0]) + } + if args[1] == "" { + t.Error("python script should not be empty") + } + tempFile := args[2] + if tempFile == "" { + t.Error("temp file path should not be empty") + } + if err := os.WriteFile(tempFile, expectedOutput, 0644); err != nil { + t.Errorf("failed to write temp file: %v", err) + } + }, + } + + python := &Python{executor: mockExecutor} + tags, err := python.CreateTagsMap() + if err != nil { + t.Fatalf("CreateTagsMap failed: %v", err) + } + + if tags["language"] != "python" { + t.Errorf("expected language tag to be 'python', got %q", tags["language"]) + } + + for key, expectedValue := range expectedPythonTags { + if actualValue, exists := tags[key]; !exists { + t.Errorf("expected tag %q to exist", key) + } else if actualValue != expectedValue { + t.Errorf("expected tag %q=%q, got %q", key, expectedValue, actualValue) + } + } +} + +func TestPython_CreateTagsMap_CommandFailure(t *testing.T) { + defer func() { _ = os.RemoveAll(constants.PlanDirectory) }() + + mockExecutor := &mockCommandExecutor{ + runErr: &exec.ExitError{}, + } + + python := &Python{executor: mockExecutor} + tags, err := python.CreateTagsMap() + + if err == nil { + t.Error("expected error when python command fails") + } + if tags != nil { + t.Error("expected nil tags when command fails") + } + + expectedPrefix := "failed to execute Python script" + if !strings.HasPrefix(err.Error(), expectedPrefix) { + t.Errorf("expected error to start with %q, got %q", expectedPrefix, err.Error()) + } +} + +func TestPython_CreateTagsMap_InvalidJSON(t *testing.T) { + defer func() { _ = os.RemoveAll(constants.PlanDirectory) }() + + invalidJSON := `{invalid json}` + mockExecutor := &mockCommandExecutor{ + onRun: func(name string, args []string, envMap map[string]string) { + if len(args) < 3 { + t.Errorf("expected at least 3 args, got %d", len(args)) + return + } + tempFile := args[2] + if err := os.WriteFile(tempFile, []byte(invalidJSON), 0644); err != nil { + t.Errorf("failed to write temp file: %v", err) + } + }, + } + + python := &Python{executor: mockExecutor} + tags, err := python.CreateTagsMap() + + if err == nil { + t.Error("expected error when JSON is invalid") + } + if tags != nil { + t.Error("expected nil tags when JSON parsing fails") + } + + if !strings.Contains(err.Error(), "failed to parse runtime tags JSON") { + t.Errorf("expected error to contain 'failed to parse runtime tags JSON', got %q", err.Error()) + } +} + +func TestPython_DetectFramework_Pytest(t *testing.T) { + viper.Reset() + viper.Set("framework", "pytest") + defer viper.Reset() + + // Ensure PYTEST_ADDOPTS is unset so GetPlatformEnv produces a deterministic value + original, existed := os.LookupEnv(pytestAddOptsEnvVar) + if existed { + _ = os.Unsetenv(pytestAddOptsEnvVar) + defer func() { _ = os.Setenv(pytestAddOptsEnvVar, original) }() + } + + python := NewPython() + fw, err := python.DetectFramework() + + if err != nil { + t.Fatalf("DetectFramework failed: %v", err) + } + if fw == nil { + t.Fatal("expected framework to be non-nil") + } + if fw.Name() != "pytest" { + t.Errorf("expected framework name 'pytest', got %q", fw.Name()) + } + + frameworkEnv := fw.GetPlatformEnv() + if frameworkEnv[pytestAddOptsEnvVar] != pytestDefaultAddOpts { + t.Errorf("expected framework platformEnv %s=%q, got %q", + pytestAddOptsEnvVar, pytestDefaultAddOpts, frameworkEnv[pytestAddOptsEnvVar]) + } +} + +func TestPython_DetectFramework_Unsupported(t *testing.T) { + viper.Reset() + viper.Set("framework", "unittest") + settings.Init() + defer func() { + viper.Reset() + settings.Init() + }() + + python := NewPython() + fw, err := python.DetectFramework() + + if err == nil { + t.Errorf("expected error for unsupported framework, but got framework: %v", fw) + return + } + if fw != nil { + t.Error("expected nil framework for unsupported framework") + } + + expectedError := "framework 'unittest' is not supported by platform 'python'" + if err.Error() != expectedError { + t.Errorf("expected error %q, got %q", expectedError, err.Error()) + } +} + +func TestPython_EmbeddedScript(t *testing.T) { + if pythonEnvScript == "" { + t.Error("embedded Python script should not be empty") + } + + expectedContent := []string{ + "import json", + "import sys", + "import platform", + "sys.argv[1]", + "json.dump", + } + + for _, expected := range expectedContent { + if !strings.Contains(pythonEnvScript, expected) { + t.Errorf("expected Python script to contain %q", expected) + } + } +} + +func TestDetectPlatform_Python(t *testing.T) { + if _, err := exec.LookPath("python"); err != nil { + t.Skip("python not in PATH — skipping integration test") + } + checkCmd := exec.Command("python", "-c", "import importlib.metadata; importlib.metadata.version('ddtrace')") + if err := checkCmd.Run(); err != nil { + t.Skip("ddtrace not installed — skipping integration test") + } + + viper.Reset() + viper.Set("platform", "python") + settings.Init() + defer func() { + viper.Reset() + settings.Init() + }() + + platform, err := DetectPlatform() + if err != nil { + t.Fatalf("DetectPlatform() unexpected error: %v", err) + } + if platform == nil { + t.Fatal("expected non-nil platform") + } + if platform.Name() != "python" { + t.Errorf("expected platform name 'python', got %q", platform.Name()) + } +} diff --git a/internal/platform/ruby_test.go b/internal/platform/ruby_test.go index 6836afc..895f198 100644 --- a/internal/platform/ruby_test.go +++ b/internal/platform/ruby_test.go @@ -413,8 +413,8 @@ func TestDetectPlatform_Ruby(t *testing.T) { func TestDetectPlatform_Unsupported(t *testing.T) { viper.Reset() - viper.Set("platform", "python") // Set BEFORE Init - settings.Init() // Re-initialize to set defaults + viper.Set("platform", "go") // Set BEFORE Init + settings.Init() // Re-initialize to set defaults defer func() { viper.Reset() settings.Init() @@ -430,7 +430,7 @@ func TestDetectPlatform_Unsupported(t *testing.T) { t.Error("expected nil platform for unsupported platform") } - expectedError := "unsupported platform: python" + expectedError := "unsupported platform: go" if err.Error() != expectedError { t.Errorf("expected error %q, got %q", expectedError, err.Error()) } diff --git a/internal/platform/scripts/python_env.py b/internal/platform/scripts/python_env.py new file mode 100644 index 0000000..6d5f571 --- /dev/null +++ b/internal/platform/scripts/python_env.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import json +import platform +import sys + +tags = { + "runtime.name": platform.python_implementation(), + "runtime.version": platform.python_version(), + "os.platform": platform.system(), + "os.architecture": platform.machine(), + "os.version": platform.release(), +} + +# Write to file path passed as argument +with open(sys.argv[1], 'w') as f: + json.dump(tags, f) diff --git a/internal/runner/worker_env.go b/internal/runner/worker_env.go index 57be687..069918a 100644 --- a/internal/runner/worker_env.go +++ b/internal/runner/worker_env.go @@ -65,6 +65,7 @@ func ensureManifestFile(workerEnv map[string]string) { if manifestFile, ok := os.LookupEnv(constants.TestOptimizationManifestFileEnvVar); ok { workerEnv[constants.TestOptimizationManifestFileEnvVar] = manifestFile + workerEnv[constants.DDTestOptimizationManifestFileEnvVar] = manifestFile return } @@ -73,4 +74,5 @@ func ensureManifestFile(workerEnv map[string]string) { manifestPath = constants.ManifestPath } workerEnv[constants.TestOptimizationManifestFileEnvVar] = manifestPath + workerEnv[constants.DDTestOptimizationManifestFileEnvVar] = manifestPath } diff --git a/internal/testoptimization/client.go b/internal/testoptimization/client.go index c838938..7976486 100644 --- a/internal/testoptimization/client.go +++ b/internal/testoptimization/client.go @@ -1,6 +1,7 @@ package testoptimization import ( + "context" "encoding/json" "log/slog" "time" @@ -146,6 +147,18 @@ func (c *DatadogClient) GetSkippableTests() map[string]bool { duration := time.Since(startTime) slog.Debug("Finished fetching skippable tests", "count", len(skippableTests), "duration", duration) + if slog.Default().Enabled(context.TODO(), slog.LevelDebug) { + i := 0 + for key := range skippableTests { + slog.Debug("Skippable test key (format: test.bundle.suite.name.params)", "key", key) + i++ + if i >= 5 { + slog.Debug("...and more skippable tests (showing first 5)") + break + } + } + } + return skippableTests } diff --git a/main.go b/main.go index 7481842..53e2ac6 100644 --- a/main.go +++ b/main.go @@ -3,11 +3,21 @@ package main import ( "log/slog" "os" + "strings" "github.com/DataDog/ddtest/internal/cmd" ) func main() { + // Configure log level. Set DDTEST_LOG_LEVEL=debug (or DD_LOG_LEVEL=debug) to + // enable debug output, which includes sample skippable/discovered test IDs. + logLevel := slog.LevelInfo + if strings.ToLower(os.Getenv("DDTEST_LOG_LEVEL")) == "debug" || + strings.ToLower(os.Getenv("DD_LOG_LEVEL")) == "debug" { + logLevel = slog.LevelDebug + } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))) + // it doesn't make sense to use ddtest without test optimization mode, // so we just enable it _ = os.Setenv("DD_CIVISIBILITY_ENABLED", "1")