diff --git a/internal/cli/release.go b/internal/cli/release.go index d8a7b06..fd0d8d7 100644 --- a/internal/cli/release.go +++ b/internal/cli/release.go @@ -718,9 +718,12 @@ func (rm *ReleaseManager) buildArtifactsJSON( continue } - // Platform-specific artifact - // Use version + platform as key to avoid overwriting different versions of the same platform - platformKey := fmt.Sprintf("%s-%s-%s", fileInfo.Version, fileInfo.OS, fileInfo.Arch) + // Platform-specific artifact. + // Include the binary name so a platform can publish more than one + // main file (Tomcat Windows zip and the service installer .exe). + // Sidecars share that name once their suffix is removed, so the + // audit and signature stay attached to the matching binary. + platformKey := fmt.Sprintf("%s-%s-%s-%s", fileInfo.Version, fileInfo.OS, fileInfo.Arch, artifactStem(originalFilename)) basePlatformKey := fmt.Sprintf("%s-%s", fileInfo.OS, fileInfo.Arch) if _, exists := platforms[platformKey]; !exists { platforms[platformKey] = &storage.PlatformArtifact{ @@ -1058,6 +1061,16 @@ func contains(slice []string, val string) bool { return false } +// artifactStem returns the main binary name for a file and its sidecars. +func artifactStem(filename string) string { + for _, suffix := range []string{".audit.json", ".metadata.json", ".sha512", ".asc", ".sig", ".cert"} { + if strings.HasSuffix(filename, suffix) { + return strings.TrimSuffix(filename, suffix) + } + } + return filename +} + // extractVersionFromFilename extracts version string from a filename. // Example: "node-v22.15.0-linux-x64.tar.xz" -> "22.15.0" func extractVersionFromFilename(filename string) string { diff --git a/internal/cli/release_test.go b/internal/cli/release_test.go index 2e93bed..bed5ef4 100644 --- a/internal/cli/release_test.go +++ b/internal/cli/release_test.go @@ -764,6 +764,98 @@ func TestBuildArtifactsJSON_RPMPackageWithAudit(t *testing.T) { } } +func TestBuildArtifactsJSON_KeepsTomcatZipAndExe(t *testing.T) { + rm := &ReleaseManager{ + db: &mockDatabaseStore{}, + github: &mockGitHubReleaser{}, + stdout: slog.New(slog.NewJSONHandler(io.Discard, nil)), + stderr: slog.New(slog.NewJSONHandler(io.Discard, nil)), + } + + uploaded := map[string]artifactInfo{ + "apache-tomcat-10.1.60-windows-x64.zip": { + URL: "https://example.com/apache-tomcat-10.1.60-windows-x64.zip", + }, + "apache-tomcat-10.1.60-windows-x64.zip.asc": { + URL: "https://example.com/apache-tomcat-10.1.60-windows-x64.zip.asc", + }, + "apache-tomcat-10.1.60-windows-x64.zip.audit.json": { + URL: "https://example.com/apache-tomcat-10.1.60-windows-x64.zip.audit.json", + }, + "apache-tomcat-10.1.60.exe": { + URL: "https://example.com/apache-tomcat-10.1.60.exe", + }, + "apache-tomcat-10.1.60.exe.asc": { + URL: "https://example.com/apache-tomcat-10.1.60.exe.asc", + }, + "apache-tomcat-10.1.60.exe.audit.json": { + URL: "https://example.com/apache-tomcat-10.1.60.exe.audit.json", + }, + "apache-tomcat-10.1.60.tar.gz": { + URL: "https://example.com/apache-tomcat-10.1.60.tar.gz", + }, + } + results := []runtime.DownloadResult{ + { + LocalPath: "/tmp/windows-x64/apache-tomcat-10.1.60-windows-x64.zip", + Platform: platform.Platform{OS: "windows", Arch: "x64"}, + Version: "10.1.60", + }, + { + LocalPath: "/tmp/windows-x64/apache-tomcat-10.1.60.exe", + Platform: platform.Platform{OS: "windows", Arch: "x64"}, + Version: "10.1.60", + }, + { + LocalPath: "/tmp/linux-x64/apache-tomcat-10.1.60.tar.gz", + Platform: platform.Platform{OS: "linux", Arch: "x64"}, + Version: "10.1.60", + }, + } + + jsonStr, err := rm.buildArtifactsJSON(uploaded, results) + if err != nil { + t.Fatalf("buildArtifactsJSON() error = %v", err) + } + + var artifacts storage.ReleaseArtifacts + if err := json.Unmarshal([]byte(jsonStr), &artifacts); err != nil { + t.Fatalf("buildArtifactsJSON() produced invalid JSON: %v", err) + } + + var zipPlat, exePlat *storage.PlatformArtifact + for i := range artifacts.Platforms { + plat := &artifacts.Platforms[i] + if plat.Binary == nil { + continue + } + switch plat.Binary.Filename { + case "apache-tomcat-10.1.60-windows-x64.zip": + zipPlat = plat + case "apache-tomcat-10.1.60.exe": + exePlat = plat + } + } + if zipPlat == nil { + t.Fatal("windows zip missing from release artifacts") + } + if exePlat == nil { + t.Fatal("windows installer missing from release artifacts") + } + if zipPlat.Platform != "windows-x64" || exePlat.Platform != "windows-x64" { + t.Fatalf("platforms = %s and %s, want windows-x64", zipPlat.Platform, exePlat.Platform) + } + if zipPlat.Signature == nil || zipPlat.Signature.Filename != "apache-tomcat-10.1.60-windows-x64.zip.asc" { + t.Errorf("zip signature = %+v", zipPlat.Signature) + } + if exePlat.Signature == nil || exePlat.Signature.Filename != "apache-tomcat-10.1.60.exe.asc" { + t.Errorf("exe signature = %+v", exePlat.Signature) + } + if zipPlat.Audit == nil || exePlat.Audit == nil { + t.Error("zip and installer each need their own audit file") + } +} + func TestUploadArtifacts_DuplicateBasenameGetsPlatformQualifiedName(t *testing.T) { tempDir := t.TempDir() linuxDir := filepath.Join(tempDir, "linux-x64") diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 275bf2f..ff71dc7 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" "time" @@ -371,37 +372,6 @@ func (m *Manager) DownloadRuntime(ctx context.Context, runtimeName string, versi return nil, fmt.Errorf("failed to get runtime provider: %w", err) } - // Check database to skip already downloaded files - if m.db != nil { - var platformsToDownload []platform.Platform - for _, plat := range platforms { - alreadyDownloaded, err := m.db.IsAlreadyDownloaded( - runtimeName, version.LatestPatch, plat.OS, plat.Arch) - if err != nil { - m.stderr.Warn("failed to check download status in database", - "runtime", runtimeName, "version", version.LatestPatch, - "platform", plat.Classifier, "error", err) - platformsToDownload = append(platformsToDownload, plat) - continue - } - - if alreadyDownloaded { - m.stdout.Info("skipping already downloaded and verified file", - "runtime", runtimeName, "version", version.LatestPatch, - "platform", plat.Classifier) - continue - } - platformsToDownload = append(platformsToDownload, plat) - } - - if len(platformsToDownload) == 0 { - m.stdout.Info("all files already downloaded, skipping", - "runtime", runtimeName, "version", version.Version) - return []DownloadResult{}, nil - } - platforms = platformsToDownload - } - // Create download tasks m.stdout.Debug("creating download tasks", "runtime", runtimeName, "version", version.Version) tasks, err := provider.CreateDownloadTasks(version, platforms, outputDir) @@ -413,6 +383,15 @@ func (m *Manager) DownloadRuntime(ctx context.Context, runtimeName string, versi return nil, fmt.Errorf("failed to create download tasks: %w", err) } + if m.db != nil { + tasks = m.withoutRecordedTasks(tasks, runtimeName, version.LatestPatch) + if len(tasks) == 0 { + m.stdout.Info("all files already downloaded, skipping", + "runtime", runtimeName, "version", version.Version) + return []DownloadResult{}, nil + } + } + m.stdout.Debug("download tasks created", "runtime", runtimeName, "version", version.Version, @@ -668,6 +647,61 @@ func (m *Manager) DownloadRuntime(ctx context.Context, runtimeName string, versi return results, nil } +// withoutRecordedTasks drops main files that already have a successful +// database row, plus the checksum and signature tasks that belong to them. +// A platform can have more than one main file. Only the recorded filename is skipped. +func (m *Manager) withoutRecordedTasks(tasks []DownloadTask, runtimeName, version string) []DownloadTask { + recorded := make(map[string]struct{}) + for _, task := range tasks { + if task.FileType != "main" { + continue + } + filename := filepath.Base(task.OutputPath) + done, err := m.db.IsFileDownloaded(runtimeName, version, task.Platform.OS, task.Platform.Arch, filename) + if err != nil { + m.stderr.Warn("failed to check download status in database", + "runtime", runtimeName, + "version", version, + "file", filename, + "error", err) + continue + } + if !done { + continue + } + recorded[task.OutputPath] = struct{}{} + m.stdout.Info("skipping already downloaded file", + "runtime", runtimeName, + "version", version, + "platform", fmt.Sprintf("%s-%s", task.Platform.OS, task.Platform.Arch), + "file", filename) + } + if len(recorded) == 0 { + return tasks + } + + kept := make([]DownloadTask, 0, len(tasks)) + for _, task := range tasks { + if taskIsRecorded(recorded, task.OutputPath) { + continue + } + kept = append(kept, task) + } + return kept +} + +func taskIsRecorded(recorded map[string]struct{}, outputPath string) bool { + if _, ok := recorded[outputPath]; ok { + return true + } + for mainPath := range recorded { + if strings.HasPrefix(outputPath, mainPath+".") { + return true + } + } + return false +} + // verifyDownloads verifies downloaded files func (m *Manager) verifyDownloads(ctx context.Context, results []DownloadResult, strategy VerificationStrategy, auditLogger *slog.Logger) { // Inject audit logger into strategy if it has a Logger field @@ -935,7 +969,7 @@ func (d *ConcurrentDownloader) downloadFile(ctx context.Context, task DownloadTa result.FileSize = size result.Size = size // Compatibility alias - d.stdout.Debug("file download completed", + d.stdout.Info("downloaded file", "url", task.URL, "output_path", task.OutputPath, "size_bytes", size, diff --git a/internal/runtime/runtime_test.go b/internal/runtime/runtime_test.go index 55b4359..fc6df38 100644 --- a/internal/runtime/runtime_test.go +++ b/internal/runtime/runtime_test.go @@ -3,6 +3,7 @@ package runtime import ( "context" "fmt" + "io" "log/slog" "net/http" "net/http/httptest" @@ -681,6 +682,50 @@ func TestManager_DownloadRuntime(t *testing.T) { } } +func TestManager_withoutRecordedTasks_KeepsUnrecordedFileOnSamePlatform(t *testing.T) { + db, err := storage.InitDB(storage.Config{DatabasePath: ":memory:", LogLevel: "silent"}) + if err != nil { + t.Fatalf("InitDB() error = %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + const version = "10.1.60" + zip := &storage.Download{ + Runtime: "tomcat", + Version: version, + Platform: "windows", + Architecture: "x64", + Filename: "apache-tomcat-10.1.60-windows-x64.zip", + SourceURL: "https://example.com/zip", + DownloadedAt: time.Now(), + VerificationStatus: "success", + } + if err := db.RecordDownload(zip); err != nil { + t.Fatalf("RecordDownload() error = %v", err) + } + + manager := NewManager(NewRegistry(), db, slog.New(slog.NewJSONHandler(io.Discard, nil)), slog.New(slog.NewJSONHandler(io.Discard, nil))) + zipPath := filepath.Join(t.TempDir(), "windows-x64", zip.Filename) + exePath := filepath.Join(filepath.Dir(zipPath), "apache-tomcat-10.1.60.exe") + tasks := []DownloadTask{ + {OutputPath: zipPath, FileType: "main", Platform: platform.Platform{OS: "windows", Arch: "x64"}}, + {OutputPath: zipPath + ".sha512", FileType: "checksum", Platform: platform.Platform{OS: "windows", Arch: "x64"}}, + {OutputPath: exePath, FileType: "main", Platform: platform.Platform{OS: "windows", Arch: "x64"}}, + {OutputPath: exePath + ".sha512", FileType: "checksum", Platform: platform.Platform{OS: "windows", Arch: "x64"}}, + {OutputPath: exePath + ".asc", FileType: "signature", Platform: platform.Platform{OS: "windows", Arch: "x64"}}, + } + + got := manager.withoutRecordedTasks(tasks, "tomcat", version) + if len(got) != 3 { + t.Fatalf("withoutRecordedTasks() len = %d, want 3", len(got)) + } + for _, task := range got { + if filepath.Base(task.OutputPath) == zip.Filename || strings.HasPrefix(task.OutputPath, zipPath+".") { + t.Errorf("recorded zip task kept: %s", task.OutputPath) + } + } +} + func TestManager_DownloadRuntime_ErrorCases(t *testing.T) { registry := NewRegistry() diff --git a/internal/runtimes/tomcat/tomcat.go b/internal/runtimes/tomcat/tomcat.go index 4b961b1..b179a63 100644 --- a/internal/runtimes/tomcat/tomcat.go +++ b/internal/runtimes/tomcat/tomcat.go @@ -273,45 +273,14 @@ func (t *TomcatAdapter) CreateDownloadTasks(version endoflife.VersionInfo, platf continue // Skip this platform gracefully } - // Create output path with platform subdirectory - platformDir := fmt.Sprintf("%s-%s", plat.OS, plat.Arch) - outputPath := filepath.Join(outputDir, platformDir, file.Filename) - - // Main binary download task - mainTask := runtime.DownloadTask{ - URL: file.DownloadURL, - OutputPath: outputPath, - Platform: plat, - Runtime: "tomcat", - Version: versionStr, - FileType: "main", - } - tasks = append(tasks, mainTask) - - // SHA512 checksum file - checksumTask := runtime.DownloadTask{ - URL: file.DownloadURL + ".sha512", - OutputPath: outputPath + ".sha512", - Platform: plat, - Runtime: "tomcat", - Version: versionStr, - FileType: "checksum", - } - tasks = append(tasks, checksumTask) - - // GPG signature file (optional) - signatureTask := runtime.DownloadTask{ - URL: file.DownloadURL + ".asc", - OutputPath: outputPath + ".asc", - Platform: plat, - Runtime: "tomcat", - Version: versionStr, - FileType: "signature", - Optional: true, - } - tasks = append(tasks, signatureTask) + tasks = append(tasks, t.tasksForFile(*file, outputDir)...) } + // The Windows service installer is one file per version + // (apache-tomcat-.exe). It is published next to the + // per-arch zip archives and does not replace them. + tasks = append(tasks, t.windowsInstallerTasks(versionStr, platforms, outputDir)...) + // Store proactive filtering results for reporting t.lastProactiveResult = &ProactiveFilterResult{ SkippedPlatforms: proactiveSkips, @@ -443,6 +412,102 @@ func (t *TomcatAdapter) ApplyPolicy(versions []endoflife.VersionInfo, policy []e // Helper functions +// tasksForFile returns the binary plus its SHA512 and GPG sidecar tasks. +func (t *TomcatAdapter) tasksForFile(file TomcatFileInfo, outputDir string) []runtime.DownloadTask { + platformDir := fmt.Sprintf("%s-%s", file.Platform.OS, file.Platform.Arch) + outputPath := filepath.Join(outputDir, platformDir, file.Filename) + + return []runtime.DownloadTask{ + { + URL: file.DownloadURL, + OutputPath: outputPath, + Platform: file.Platform, + Runtime: "tomcat", + Version: file.Version, + FileType: "main", + }, + { + URL: file.DownloadURL + ".sha512", + OutputPath: outputPath + ".sha512", + Platform: file.Platform, + Runtime: "tomcat", + Version: file.Version, + FileType: "checksum", + }, + { + URL: file.DownloadURL + ".asc", + OutputPath: outputPath + ".asc", + Platform: file.Platform, + Runtime: "tomcat", + Version: file.Version, + FileType: "signature", + Optional: true, + }, + } +} + +// windowsInstallerTasks adds the Windows service installer when a Windows +// platform was requested and Apache published the .exe. A missing installer +// leaves the zip and tar.gz tasks in place. +func (t *TomcatAdapter) windowsInstallerTasks(version string, platforms []platform.Platform, outputDir string) []runtime.DownloadTask { + plat, ok := preferredWindowsPlatform(platforms) + if !ok { + return nil + } + + filename := fmt.Sprintf("apache-tomcat-%s.exe", version) + downloadURL := fmt.Sprintf("%s/tomcat-%s/v%s/bin/%s", t.archiveBaseURL(), t.extractMajorVersion(version), version, filename) + if !t.checkFileExists(downloadURL) { + t.stderr.Warn("Tomcat Windows installer not found", + "version", version, + "url", downloadURL) + return nil + } + + plat.FileExt = "exe" + t.stdout.Info("including Tomcat Windows installer", + "version", version, + "filename", filename, + "platform", fmt.Sprintf("%s-%s", plat.OS, plat.Arch)) + + return t.tasksForFile(TomcatFileInfo{ + Filename: filename, + DownloadURL: downloadURL, + Platform: plat, + Version: version, + Exists: true, + }, outputDir) +} + +// preferredWindowsPlatform returns the Windows platform the installer is +// recorded against. x64 wins when both Windows architectures were requested +// so the single installer is downloaded once. +func preferredWindowsPlatform(platforms []platform.Platform) (platform.Platform, bool) { + var fallback platform.Platform + found := false + for _, plat := range platforms { + if plat.OS != "windows" { + continue + } + if plat.Arch == "x64" { + return plat, true + } + if !found { + fallback = plat + found = true + } + } + return fallback, found +} + +func (t *TomcatAdapter) archiveBaseURL() string { + baseURL := "https://archive.apache.org/dist/tomcat" + if t.config != nil && t.config.Download.BaseURL != "" { + baseURL = t.config.Download.BaseURL + } + return strings.TrimRight(baseURL, "/") +} + // getFilename returns the appropriate filename for the platform func (t *TomcatAdapter) getFilename(version string, plat platform.Platform) string { if plat.OS == "windows" { @@ -527,10 +592,7 @@ func (t *TomcatAdapter) getAvailableFiles(version string) (*TomcatReleaseInfo, e allPlatforms := t.getAllPossiblePlatforms() // Check each platform to see if files actually exist - baseURL := "https://archive.apache.org/dist/tomcat" // fallback when config not set - if t.config != nil && t.config.Download.BaseURL != "" { - baseURL = t.config.Download.BaseURL - } + baseURL := t.archiveBaseURL() for _, plat := range allPlatforms { filename := t.getFilename(version, plat) downloadURL := fmt.Sprintf("%s/tomcat-%s/v%s/bin/%s", baseURL, majorVersion, version, filename) diff --git a/internal/runtimes/tomcat/tomcat_test.go b/internal/runtimes/tomcat/tomcat_test.go new file mode 100644 index 0000000..7ca334f --- /dev/null +++ b/internal/runtimes/tomcat/tomcat_test.go @@ -0,0 +1,181 @@ +package tomcat + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path" + "testing" + + "github.com/clean-dependency-project/cdprun/internal/config" + "github.com/clean-dependency-project/cdprun/internal/endoflife" + "github.com/clean-dependency-project/cdprun/internal/platform" + "github.com/clean-dependency-project/cdprun/internal/runtime" +) + +func TestCreateDownloadTasks_IncludesWindowsInstallerAndZip(t *testing.T) { + const version = "10.1.60" + available := map[string]bool{ + "apache-tomcat-10.1.60-windows-x64.zip": true, + "apache-tomcat-10.1.60-windows-x86.zip": true, + "apache-tomcat-10.1.60.tar.gz": true, + "apache-tomcat-10.1.60.exe": true, + } + adapter := newArchiveAdapter(t, available) + + tasks, err := adapter.CreateDownloadTasks(endoflife.VersionInfo{Version: version}, []platform.Platform{ + {OS: "windows", Arch: "x64"}, + {OS: "windows", Arch: "x86"}, + {OS: "linux", Arch: "x64"}, + }, t.TempDir()) + if err != nil { + t.Fatalf("CreateDownloadTasks() error = %v", err) + } + + mains := mainFilenames(tasks) + for name := range available { + if mains[name] != 1 { + t.Errorf("main task %s count = %d, want 1", name, mains[name]) + } + } + if mains["apache-tomcat-10.1.60.exe"] != 1 { + t.Fatalf("installer downloaded %d times, want 1", mains["apache-tomcat-10.1.60.exe"]) + } + + for _, task := range tasks { + if task.FileType != "main" || path.Base(task.OutputPath) != "apache-tomcat-10.1.60.exe" { + continue + } + if task.Platform.OS != "windows" || task.Platform.Arch != "x64" { + t.Errorf("installer platform = %s-%s, want windows-x64", task.Platform.OS, task.Platform.Arch) + } + if !hasSidecar(tasks, task.OutputPath+".sha512", "checksum") || !hasSidecar(tasks, task.OutputPath+".asc", "signature") { + t.Error("installer is missing checksum or signature task") + } + } +} + +func TestCreateDownloadTasks_MissingInstallerKeepsZip(t *testing.T) { + available := map[string]bool{ + "apache-tomcat-11.0.26-windows-x64.zip": true, + "apache-tomcat-11.0.26.tar.gz": true, + } + adapter := newArchiveAdapter(t, available) + + tasks, err := adapter.CreateDownloadTasks(endoflife.VersionInfo{Version: "11.0.26"}, []platform.Platform{ + {OS: "windows", Arch: "x64"}, + {OS: "linux", Arch: "x64"}, + }, t.TempDir()) + if err != nil { + t.Fatalf("CreateDownloadTasks() error = %v", err) + } + + mains := mainFilenames(tasks) + if mains["apache-tomcat-11.0.26-windows-x64.zip"] != 1 { + t.Errorf("windows zip count = %d, want 1", mains["apache-tomcat-11.0.26-windows-x64.zip"]) + } + if mains["apache-tomcat-11.0.26.tar.gz"] != 1 { + t.Errorf("tar.gz count = %d, want 1", mains["apache-tomcat-11.0.26.tar.gz"]) + } + if mains["apache-tomcat-11.0.26.exe"] != 0 { + t.Errorf("installer count = %d, want 0", mains["apache-tomcat-11.0.26.exe"]) + } +} + +func TestCreateDownloadTasks_LinuxOnlySkipsInstaller(t *testing.T) { + var sawExe bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if path.Base(r.URL.Path) == "apache-tomcat-9.0.122.exe" { + sawExe = true + } + if path.Base(r.URL.Path) == "apache-tomcat-9.0.122.tar.gz" { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + t.Cleanup(server.Close) + + adapter := NewAdapterWithConfig(nil, &config.Runtime{ + Download: config.DownloadConfig{BaseURL: server.URL}, + }, nil, discardLogger(), discardLogger()) + + tasks, err := adapter.CreateDownloadTasks(endoflife.VersionInfo{Version: "9.0.122"}, []platform.Platform{ + {OS: "linux", Arch: "x64"}, + }, t.TempDir()) + if err != nil { + t.Fatalf("CreateDownloadTasks() error = %v", err) + } + if sawExe { + t.Error("linux-only download probed the Windows installer") + } + if mainFilenames(tasks)["apache-tomcat-9.0.122.tar.gz"] != 1 { + t.Error("linux tar.gz task missing") + } +} + +func TestCreateDownloadTasks_InstallerUsesX86WhenX64NotRequested(t *testing.T) { + available := map[string]bool{ + "apache-tomcat-9.0.122-windows-x86.zip": true, + "apache-tomcat-9.0.122.exe": true, + } + adapter := newArchiveAdapter(t, available) + + tasks, err := adapter.CreateDownloadTasks(endoflife.VersionInfo{Version: "9.0.122"}, []platform.Platform{ + {OS: "windows", Arch: "x86"}, + }, t.TempDir()) + if err != nil { + t.Fatalf("CreateDownloadTasks() error = %v", err) + } + + for _, task := range tasks { + if task.FileType == "main" && path.Base(task.OutputPath) == "apache-tomcat-9.0.122.exe" { + if task.Platform.Arch != "x86" { + t.Errorf("installer arch = %s, want x86", task.Platform.Arch) + } + return + } + } + t.Fatal("installer task not created") +} + +func newArchiveAdapter(t *testing.T, available map[string]bool) *TomcatAdapter { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if available[path.Base(r.URL.Path)] { + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + t.Cleanup(server.Close) + + return NewAdapterWithConfig(nil, &config.Runtime{ + Download: config.DownloadConfig{BaseURL: server.URL}, + }, nil, discardLogger(), discardLogger()) +} + +func mainFilenames(tasks []runtime.DownloadTask) map[string]int { + counts := make(map[string]int) + for _, task := range tasks { + if task.FileType != "main" { + continue + } + counts[path.Base(task.OutputPath)]++ + } + return counts +} + +func hasSidecar(tasks []runtime.DownloadTask, outputPath, fileType string) bool { + for _, task := range tasks { + if task.FileType == fileType && task.OutputPath == outputPath { + return true + } + } + return false +} + +func discardLogger() *slog.Logger { + return slog.New(slog.NewJSONHandler(io.Discard, nil)) +} diff --git a/internal/sitegen/builder.go b/internal/sitegen/builder.go index d1f719e..d1b37f5 100644 --- a/internal/sitegen/builder.go +++ b/internal/sitegen/builder.go @@ -1,7 +1,9 @@ package sitegen import ( + "fmt" "sort" + "time" ) // BuildModel transforms releases into a SiteModel with deterministic sorting. @@ -43,7 +45,7 @@ func BuildModel(releases []ReleaseWithArtifacts) *SiteModel { } patch := rel.Release.SemverPatch - runtimeMap[runtime][os][major][minor][patch] = append( + runtimeMap[runtime][os][major][minor][patch] = appendReleaseOnce( runtimeMap[runtime][os][major][minor][patch], rel, ) @@ -227,6 +229,34 @@ func buildReleaseModelForOS(rel ReleaseWithArtifacts, osFilter string) ReleaseMo } } +// appendReleaseOnce records a release once per OS version bucket. +// One release can carry several binaries for the same OS (Windows zip and exe). +// Listing it once keeps the page from repeating that whole set. +func appendReleaseOnce(listed []ReleaseWithArtifacts, rel ReleaseWithArtifacts) []ReleaseWithArtifacts { + key := releaseIdentity(rel) + for _, existing := range listed { + if releaseIdentity(existing) == key { + return listed + } + } + return append(listed, rel) +} + +func releaseIdentity(rel ReleaseWithArtifacts) string { + if rel.Release.ID != 0 { + return fmt.Sprintf("id:%d", rel.Release.ID) + } + if rel.Release.ReleaseTag != "" { + return "tag:" + rel.Release.ReleaseTag + } + return fmt.Sprintf("%s|%s|%s|%s", + rel.Release.Runtime, + rel.Release.Version, + rel.Release.ReleaseURL, + rel.Release.CreatedAt.UTC().Format(time.RFC3339Nano), + ) +} + // normalizeOS normalizes OS names for consistent sorting. // Maps: "darwin" -> "mac", others unchanged. func normalizeOS(os string) string { diff --git a/internal/sitegen/sitegen_test.go b/internal/sitegen/sitegen_test.go index c7fd4ce..03cbddc 100644 --- a/internal/sitegen/sitegen_test.go +++ b/internal/sitegen/sitegen_test.go @@ -14,6 +14,57 @@ import ( "github.com/clean-dependency-project/cdprun/internal/storage" ) +func TestBuildModel_ListsWindowsZipAndExeOnce(t *testing.T) { + now := time.Now() + model := BuildModel([]ReleaseWithArtifacts{ + { + Release: storage.Release{ + Runtime: "tomcat", + Version: "10.1.60", + SemverMajor: 10, + SemverMinor: 1, + SemverPatch: 60, + ReleaseTag: "tomcat-multi-test", + CreatedAt: now, + }, + Artifacts: storage.ReleaseArtifacts{ + Platforms: []storage.PlatformArtifact{ + { + Platform: "windows-x64", + PlatformOS: "windows", + PlatformArch: "x64", + Binary: &storage.ArtifactFile{ + Filename: "apache-tomcat-10.1.60-windows-x64.zip", + URL: "https://example.com/zip", + }, + }, + { + Platform: "windows-x64", + PlatformOS: "windows", + PlatformArch: "x64", + Binary: &storage.ArtifactFile{ + Filename: "apache-tomcat-10.1.60.exe", + URL: "https://example.com/exe", + }, + }, + }, + }, + }, + }) + + if len(model.Runtimes) != 1 || len(model.Runtimes[0].Platforms) != 1 { + t.Fatalf("model = %+v", model) + } + versions := model.Runtimes[0].Platforms[0].Versions + if len(versions) != 1 || len(versions[0].Releases) != 1 { + t.Fatalf("versions = %+v, want one release", versions) + } + artifacts := versions[0].Releases[0].Artifacts + if len(artifacts) != 2 { + t.Fatalf("artifacts = %d, want zip and exe", len(artifacts)) + } +} + func TestNormalizePackageName(t *testing.T) { tests := []struct { name string @@ -1836,7 +1887,7 @@ func TestExpandUnsupportedVersions(t *testing.T) { // Lexicographic would give: "10","10.24.1","16","16.20.1","16.20.2","8","8.17.0" // Correct numeric: "8","8.17.0","10","10.24.1","16","16.20.1","16.20.2" wantVersions: []string{"8", "8.17.0", "10", "10.24.1", "16", "16.20.1", "16.20.2"}, - }, { + }, { name: "exact version match emits only the concrete version (no prefix duplicate)", uc: config.UnsupportedConfig{ "nodejs": {{Version: "18.20.0", Reason: "EOL", EOLDate: "2024-04-30"}}, diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 85ebd3e..1ce7425 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -24,14 +24,16 @@ type Download struct { ID uint `gorm:"primaryKey"` // What was downloaded - Runtime string `gorm:"not null;index:idx_runtime_version;uniqueIndex:idx_unique_download"` - Version string `gorm:"not null;index:idx_runtime_version,idx_version;uniqueIndex:idx_unique_download"` - VersionMajor int `gorm:"index"` - VersionMinor int - VersionPatch int - Platform string `gorm:"not null;index:idx_platform;uniqueIndex:idx_unique_download"` - Architecture string `gorm:"not null;index:idx_platform;uniqueIndex:idx_unique_download"` - Filename string `gorm:"not null"` + Runtime string `gorm:"not null;index:idx_runtime_version;uniqueIndex:idx_unique_download"` + Version string `gorm:"not null;index:idx_runtime_version,idx_version;uniqueIndex:idx_unique_download"` + VersionMajor int `gorm:"index"` + VersionMinor int + VersionPatch int + Platform string `gorm:"not null;index:idx_platform;uniqueIndex:idx_unique_download"` + Architecture string `gorm:"not null;index:idx_platform;uniqueIndex:idx_unique_download"` + // Filename is part of the identity so one platform can store more than one + // artifact, such as the Tomcat Windows zip and the service installer exe. + Filename string `gorm:"not null;uniqueIndex:idx_unique_download"` FileExtension string FileSize int64 SourceURL string `gorm:"not null"` @@ -82,6 +84,7 @@ type Store interface { RecordDownload(*Download) error GetDownload(runtime, version, platform, arch string) (*Download, error) IsAlreadyDownloaded(runtime, version, platform, arch string) (bool, error) + IsFileDownloaded(runtime, version, platform, arch, filename string) (bool, error) UpdateVerification(id uint, checksumVerified, gpgVerified bool, status, errorMsg string) error UpdateChecksumVerification(id uint, verified bool, algorithm, value, sourceURL string) error UpdateGPGVerification(id uint, verified bool, signatureURL, keyringSource string) error @@ -129,10 +132,27 @@ func InitDB(cfg Config) (*DB, error) { if err := db.AutoMigrate(&Download{}, &Release{}, &PackageRecord{}); err != nil { return nil, fmt.Errorf("failed to migrate schema: %w", err) } + if err := ensureDownloadUniqueIndex(db); err != nil { + return nil, err + } return &DB{db: db}, nil } +// ensureDownloadUniqueIndex rebuilds the download identity index so it +// includes filename. AutoMigrate leaves an existing unique index in place, +// and the previous index rejected a second artifact for the same platform. +func ensureDownloadUniqueIndex(db *gorm.DB) error { + if err := db.Exec("DROP INDEX IF EXISTS idx_unique_download").Error; err != nil { + return fmt.Errorf("drop download unique index: %w", err) + } + if err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_download + ON downloads (runtime, version, platform, architecture, filename)`).Error; err != nil { + return fmt.Errorf("create download unique index: %w", err) + } + return nil +} + // Close closes the database connection func (d *DB) Close() error { sqlDB, err := d.db.DB() @@ -182,6 +202,19 @@ func (d *DB) IsAlreadyDownloaded(runtime, version, platform, arch string) (bool, return count > 0, nil } +// IsFileDownloaded reports whether this exact artifact was recorded as a +// successful download. A second file for the same platform is a different row. +func (d *DB) IsFileDownloaded(runtime, version, platform, arch, filename string) (bool, error) { + var count int64 + err := d.db.Model(&Download{}).Where( + "runtime = ? AND version = ? AND platform = ? AND architecture = ? AND filename = ? AND verification_status = ?", + runtime, version, platform, arch, filename, "success").Count(&count).Error + if err != nil { + return false, fmt.Errorf("failed to check if file is already downloaded: %w", err) + } + return count > 0, nil +} + // UpdateVerification updates verification status for a download func (d *DB) UpdateVerification(id uint, checksumVerified, gpgVerified bool, status, errorMsg string) error { updates := map[string]interface{}{ diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index c20004e..b5949ed 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -241,12 +241,12 @@ func TestRecordDownload(t *testing.T) { Version: "18.20.8", Platform: "darwin", Architecture: "arm64", - Filename: "duplicate.tar.gz", + Filename: "node-v18.20.8-darwin-arm64.tar.gz", SourceURL: "https://example.com/duplicate.tar.gz", DownloadedAt: time.Now(), VerificationStatus: "success", }, - wantError: true, // Will fail on second insert due to unique constraint + wantError: true, // Same runtime, version, platform, arch, and filename }, } @@ -288,6 +288,38 @@ func TestRecordDownload(t *testing.T) { } // TestGetDownload tests retrieving download entries +func TestRecordDownload_AllowsDistinctFilesOnSamePlatform(t *testing.T) { + db := newTestDB(t) + + zipDownload := createTestDownload("tomcat", "10.1.60", "windows", "x64") + zipDownload.Filename = "apache-tomcat-10.1.60-windows-x64.zip" + exeDownload := createTestDownload("tomcat", "10.1.60", "windows", "x64") + exeDownload.Filename = "apache-tomcat-10.1.60.exe" + + if err := db.RecordDownload(zipDownload); err != nil { + t.Fatalf("RecordDownload(zip) error = %v", err) + } + if err := db.RecordDownload(exeDownload); err != nil { + t.Fatalf("RecordDownload(exe) error = %v", err) + } + + zipDone, err := db.IsFileDownloaded("tomcat", "10.1.60", "windows", "x64", zipDownload.Filename) + if err != nil { + t.Fatalf("IsFileDownloaded(zip) error = %v", err) + } + exeDone, err := db.IsFileDownloaded("tomcat", "10.1.60", "windows", "x64", exeDownload.Filename) + if err != nil { + t.Fatalf("IsFileDownloaded(exe) error = %v", err) + } + missing, err := db.IsFileDownloaded("tomcat", "10.1.60", "windows", "x64", "missing.exe") + if err != nil { + t.Fatalf("IsFileDownloaded(missing) error = %v", err) + } + if !zipDone || !exeDone || missing { + t.Fatalf("recorded zip=%v exe=%v missing=%v", zipDone, exeDone, missing) + } +} + func TestGetDownload(t *testing.T) { tests := []struct { name string