diff --git a/pkg/runtime/python/build.go b/pkg/runtime/python/build.go index 7cbd1556cb..35447a40e7 100644 --- a/pkg/runtime/python/build.go +++ b/pkg/runtime/python/build.go @@ -5,6 +5,7 @@ import ( "archive/zip" "compress/gzip" "context" + "crypto/sha256" "encoding/json" "fmt" "io" @@ -78,12 +79,10 @@ func buildDeploy(ctx context.Context, input *runtime.BuildInput, cacheDir string return nil, fmt.Errorf("package discovery: %w", err) } - var packagesBuilt []string for _, pkg := range localPackages { if err := buildPackage(ctx, input, pkg); err != nil { return nil, fmt.Errorf("build %s: %w", pkg.Name, err) } - packagesBuilt = append(packagesBuilt, pkg.Name) } if err := installDependenciesForBuild(ctx, input, projectInfo); err != nil { @@ -185,11 +184,14 @@ func precompilePythonFiles(ctx context.Context, input *runtime.BuildInput, dir s func ensureDockerfile(input *runtime.BuildInput, projectInfo *projectInfo) error { outputDockerfile := filepath.Join(input.Out(), "Dockerfile") - // Ensure pyproject.toml is in the build context for `pip install .` + // Preserve the handler package metadata for custom Dockerfiles that build it. + // Default container installs use the rewritten requirements.txt instead. outputPyproject := filepath.Join(input.Out(), "pyproject.toml") if _, err := os.Stat(outputPyproject); err != nil && projectInfo.PyprojectPath != "" { if _, err := os.Stat(projectInfo.PyprojectPath); err == nil { - _ = copyFile(projectInfo.PyprojectPath, outputPyproject) + if err := copyFile(projectInfo.PyprojectPath, outputPyproject); err != nil { + return fmt.Errorf("copy pyproject.toml: %w", err) + } } } @@ -641,8 +643,8 @@ func installDependenciesForLambda(ctx context.Context, input *runtime.BuildInput // Container builds: Dockerfile handles deps; zip builds: install here if input.IsContainer { - if err := copyWorkspacePackagesForContainer(input, projectInfo); err != nil { - return fmt.Errorf("failed to copy workspace packages for container: %w", err) + if err := materializeContainerRequirements(ctx, input, projectInfo); err != nil { + return fmt.Errorf("failed to materialize container requirements: %w", err) } } else { if err := copySyncedDependencies(ctx, input, projectInfo, architecture); err != nil { @@ -653,76 +655,117 @@ func installDependenciesForLambda(ctx context.Context, input *runtime.BuildInput return nil } -// copyWorkspacePackagesForContainer copies workspace package directories into the artifact -// so the Dockerfile's `uv pip install -r requirements.txt` can resolve relative paths. -func copyWorkspacePackagesForContainer(input *runtime.BuildInput, projectInfo *projectInfo) error { - workspaceRoot := findWorkspaceRoot(projectInfo) - +// materializeContainerRequirements replaces checkout-relative local requirements with +// sdists stored inside the Docker build context. +func materializeContainerRequirements(ctx context.Context, input *runtime.BuildInput, projectInfo *projectInfo) error { requirementsPath := filepath.Join(input.Out(), "requirements.txt") content, err := os.ReadFile(requirementsPath) if err != nil { - return nil + return fmt.Errorf("read requirements: %w", err) } - lines := strings.Split(string(content), "\n") + rewritten, err := rewriteContainerRequirements( + ctx, + string(content), + findWorkspaceRoot(projectInfo), + input.Out(), + buildContainerSdist, + ) + if err != nil { + return err + } - for _, line := range lines { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "-") { - continue - } + return os.WriteFile(requirementsPath, []byte(rewritten), 0644) +} - if !strings.HasPrefix(line, "./") && !strings.HasPrefix(line, "../") { +type containerSdistBuilder func(context.Context, string, string) (string, error) + +func rewriteContainerRequirements( + ctx context.Context, + requirements string, + workspaceRoot string, + artifactRoot string, + buildSdist containerSdistBuilder, +) (string, error) { + lines := strings.Split(requirements, "\n") + artifacts := make(map[string]string) + + for index, rawLine := range lines { + localPath, suffix, ok := splitLocalRequirement(rawLine) + if !ok { continue } - // Strip extras or markers (e.g., "./core[extra] ; python_version >= '3.11'") - pkgPath := line - for _, sep := range []string{" ", "[", ";"} { - if idx := strings.Index(pkgPath, sep); idx > 0 { - pkgPath = pkgPath[:idx] - } + fullPath := filepath.Clean(filepath.Join(workspaceRoot, localPath)) + info, err := os.Stat(fullPath) + if err != nil { + return "", fmt.Errorf("workspace package %q: %w", localPath, err) } - - // Resolve full path relative to workspace root - fullPath := filepath.Join(workspaceRoot, pkgPath) - if _, err := os.Stat(fullPath); err != nil { - slog.Warn("workspace package directory not found", "path", fullPath, "line", line) - continue + if !info.IsDir() { + return "", fmt.Errorf("workspace package %q is not a directory", localPath) } - // Copy to artifact at the same relative path - destPath := filepath.Join(input.Out(), pkgPath) - if _, err := os.Stat(destPath); err == nil { - // Already exists — just ensure pyproject.toml is present for uv pip install - srcPyproject := filepath.Join(fullPath, "pyproject.toml") - destPyproject := filepath.Join(destPath, "pyproject.toml") - if _, err := os.Stat(srcPyproject); err == nil { - if _, err := os.Stat(destPyproject); err != nil { - data, readErr := os.ReadFile(srcPyproject) - if readErr != nil { - return fmt.Errorf("failed to read pyproject.toml for workspace package %s: %w", pkgPath, readErr) - } - if err := os.WriteFile(destPyproject, data, 0644); err != nil { - return fmt.Errorf("failed to copy pyproject.toml for workspace package %s: %w", pkgPath, err) - } - } + artifactPath, exists := artifacts[fullPath] + if !exists { + artifactPath, err = buildSdist(ctx, artifactRoot, fullPath) + if err != nil { + return "", fmt.Errorf("build workspace package %q: %w", localPath, err) } - continue + artifacts[fullPath] = artifactPath } - if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - return fmt.Errorf("failed to create directory for workspace package %s: %w", pkgPath, err) + relativePath, err := filepath.Rel(artifactRoot, artifactPath) + if err != nil { + return "", fmt.Errorf("resolve artifact path for workspace package %q: %w", localPath, err) } + if relativePath == ".." || strings.HasPrefix(relativePath, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("workspace package %q produced an artifact outside the build context", localPath) + } + lines[index] = "./" + filepath.ToSlash(relativePath) + suffix + } + + return strings.Join(lines, "\n"), nil +} + +// splitLocalRequirement separates a uv-exported local path from extras and markers. +func splitLocalRequirement(line string) (path string, suffix string, ok bool) { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "./") && !strings.HasPrefix(trimmed, "../") { + return "", "", false + } - // Preserve pyproject.toml and metadata for uv pip install - if err := copyDir(fullPath, destPath, skipBuildArtifacts); err != nil { - return fmt.Errorf("failed to copy workspace package %s: %w", pkgPath, err) + end := len(trimmed) + for _, separator := range []string{" ", "[", ";"} { + if index := strings.Index(trimmed, separator); index >= 0 && index < end { + end = index } + } + return trimmed[:end], trimmed[end:], true +} +func buildContainerSdist(ctx context.Context, artifactRoot string, packageDir string) (string, error) { + sum := sha256.Sum256([]byte(packageDir)) + outputDir := filepath.Join(artifactRoot, ".sst", "packages", fmt.Sprintf("%x", sum[:8])) + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("create package artifact directory: %w", err) } - return nil + if err := runUvBuild(ctx, &uvBuildCommand{ + PackageDir: packageDir, + OutputDir: outputDir, + BuildType: "sdist", + }); err != nil { + return "", err + } + + archives, err := filepath.Glob(filepath.Join(outputDir, "*.tar.gz")) + if err != nil { + return "", fmt.Errorf("find source distribution: %w", err) + } + if len(archives) != 1 { + return "", fmt.Errorf("expected one source distribution, found %d", len(archives)) + } + return archives[0], nil } // copySourceFilesSimple copies handler source files to the build output. diff --git a/pkg/runtime/python/build_test.go b/pkg/runtime/python/build_test.go index 851831ab54..4d2974a4d9 100644 --- a/pkg/runtime/python/build_test.go +++ b/pkg/runtime/python/build_test.go @@ -1,6 +1,7 @@ package python import ( + "context" "os" "path/filepath" "strings" @@ -9,6 +10,140 @@ import ( "github.com/sst/sst/v3/pkg/runtime" ) +func TestRewriteContainerRequirements(t *testing.T) { + t.Run("materializes parent workspace members inside the artifact", func(t *testing.T) { + root := t.TempDir() + workspaceRoot := filepath.Join(root, "projects", "app") + libDir := filepath.Join(root, "lib") + artifactRoot := filepath.Join(root, "artifact") + for _, dir := range []string{workspaceRoot, libDir, artifactRoot} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + + calls := 0 + rewritten, err := rewriteContainerRequirements( + context.Background(), + "../../lib[extra] ; python_version >= '3.12'\nrequests==2.32.0\n../../lib", + workspaceRoot, + artifactRoot, + func(_ context.Context, artifactRoot string, packageDir string) (string, error) { + calls++ + if packageDir != libDir { + t.Fatalf("package dir = %s, want %s", packageDir, libDir) + } + archive := filepath.Join(artifactRoot, ".sst", "packages", "lib", "lib-0.1.0.tar.gz") + if err := os.MkdirAll(filepath.Dir(archive), 0755); err != nil { + return "", err + } + return archive, os.WriteFile(archive, nil, 0644) + }, + ) + if err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("sdist builder called %d times, want 1", calls) + } + if strings.Contains(rewritten, "..") { + t.Fatalf("rewritten requirements contain a parent path:\n%s", rewritten) + } + if !strings.Contains(rewritten, "./.sst/packages/lib/lib-0.1.0.tar.gz[extra] ; python_version >= '3.12'") { + t.Fatalf("local requirement was not rewritten with its suffix:\n%s", rewritten) + } + if !strings.Contains(rewritten, "requests==2.32.0") { + t.Fatalf("registry requirement was changed:\n%s", rewritten) + } + }) + + t.Run("supports descendant path dependencies", func(t *testing.T) { + workspaceRoot := t.TempDir() + packageDir := filepath.Join(workspaceRoot, "packages", "common") + artifactRoot := filepath.Join(workspaceRoot, "artifact") + for _, dir := range []string{packageDir, artifactRoot} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + + rewritten, err := rewriteContainerRequirements( + context.Background(), + "./packages/common", + workspaceRoot, + artifactRoot, + func(_ context.Context, artifactRoot string, packageDir string) (string, error) { + archive := filepath.Join(artifactRoot, ".sst", "packages", "common", "common-0.1.0.tar.gz") + return archive, nil + }, + ) + if err != nil { + t.Fatal(err) + } + if rewritten != "./.sst/packages/common/common-0.1.0.tar.gz" { + t.Fatalf("rewritten requirement = %q", rewritten) + } + }) + + t.Run("rejects missing local packages", func(t *testing.T) { + _, err := rewriteContainerRequirements( + context.Background(), + "../missing", + t.TempDir(), + t.TempDir(), + func(context.Context, string, string) (string, error) { + t.Fatal("sdist builder should not be called") + return "", nil + }, + ) + if err == nil { + t.Fatal("expected missing package error") + } + }) + + t.Run("rejects archives outside the artifact", func(t *testing.T) { + workspaceRoot := t.TempDir() + packageDir := filepath.Join(workspaceRoot, "package") + if err := os.MkdirAll(packageDir, 0755); err != nil { + t.Fatal(err) + } + + _, err := rewriteContainerRequirements( + context.Background(), + "./package", + workspaceRoot, + t.TempDir(), + func(context.Context, string, string) (string, error) { + return filepath.Join(workspaceRoot, "outside.tar.gz"), nil + }, + ) + if err == nil { + t.Fatal("expected artifact containment error") + } + }) +} + +func TestSplitLocalRequirement(t *testing.T) { + tests := []struct { + line string + wantPath string + wantSuffix string + ok bool + }{ + {"./package", "./package", "", true}, + {"../../lib[extra] ; python_version >= '3.12'", "../../lib", "[extra] ; python_version >= '3.12'", true}, + {"requests==2.32.0", "", "", false}, + {"-e ./package", "", "", false}, + } + + for _, tt := range tests { + path, suffix, ok := splitLocalRequirement(tt.line) + if path != tt.wantPath || suffix != tt.wantSuffix || ok != tt.ok { + t.Errorf("splitLocalRequirement(%q) = (%q, %q, %v), want (%q, %q, %v)", tt.line, path, suffix, ok, tt.wantPath, tt.wantSuffix, tt.ok) + } + } +} + func TestDeployBuilder_CleanupInstalledDependencies(t *testing.T) { tempDir := t.TempDir() diff --git a/platform/functions/docker/python.Dockerfile b/platform/functions/docker/python.Dockerfile index 4dc049b1b5..ecdf87eb6a 100644 --- a/platform/functions/docker/python.Dockerfile +++ b/platform/functions/docker/python.Dockerfile @@ -16,14 +16,13 @@ RUN if command -v dnf > /dev/null 2>&1; then \ yum install -y git gcc python3-devel && yum clean all; \ fi -# Copy everything first so workspace packages (referenced as ./pkg in requirements.txt) -# are available during dependency installation. +# Copy everything first so local source distributions in .sst/packages are available +# during dependency installation. # # NOTE: This copies source code before installing deps, which means any code change # invalidates Docker's layer cache for the pip install step. This is a deliberate -# tradeoff — workspace packages must be present for `uv pip install` to resolve -# relative path dependencies (e.g. ./shared, ./core). Users who need better caching -# should provide a custom Dockerfile that copies requirements.txt first. +# tradeoff — local package artifacts must be present for `uv pip install`. Users who +# need better caching should copy requirements.txt and .sst/packages/ before this step. COPY . ${LAMBDA_TASK_ROOT} # Install dependencies inside the container to ensure native binaries diff --git a/platform/src/components/aws/function.ts b/platform/src/components/aws/function.ts index 1da49a2666..b56ab864f0 100644 --- a/platform/src/components/aws/function.ts +++ b/platform/src/components/aws/function.ts @@ -1160,9 +1160,19 @@ export interface FunctionArgs { * This needs the Docker daemon to be running. * ::: * - * To use a custom Dockerfile, add one to the rooot of the uv workspace + * To use a custom Dockerfile, add one to the root of the uv workspace * of the function. * + * For a cache-friendly dependency layer, copy both `requirements.txt` and the + * SST-generated `.sst/packages/` directory before installing dependencies. + * + * ```dockerfile + * COPY requirements.txt ${LAMBDA_TASK_ROOT}/requirements.txt + * COPY .sst/packages/ ${LAMBDA_TASK_ROOT}/.sst/packages/ + * RUN uv pip install -r requirements.txt --target ${LAMBDA_TASK_ROOT} --system + * COPY . ${LAMBDA_TASK_ROOT} + * ``` + * * * ```txt {5} * ├── sst.config.ts