Feat/gitlab release scan

.devcontainer/devcontainer.json

@@ -5,7 +5,8 @@
     "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
     "ghcr.io/devcontainers/features/python:1": {
       "version": "3.12"
-    }
+    },
+    "ghcr.io/devcontainers/features/github-cli:1": {}
   },
   "customizations": {
     "vscode": {

docs/guides/gitlab.md

@@ -112,6 +112,26 @@ trufflehog gitlab --token=glpat-[redacted]
 
 Note: this only scans repositories you have access to. You can specify single repositories as well.
 
+## Secret Detection in Release Assets
+
+Use `gl release scan` to scan published release assets with the same project
+filters and secret-detection controls as `gl scan`.
+
+```bash
+pipeleek gl release scan --url https://gitlab.example.com --token [redacted]
+
+pipeleek gl release scan --url https://gitlab.example.com --token [redacted] \
+  --repo group/project --tag v1.0 --include-sources
+```
+
+`--tag` requires `--repo`; alternatively, `--release-limit N` scans the newest N
+published releases per project (`0` means all).
+`--include-sources` adds one generated source ZIP per release.
+Downloads send the token only to the configured GitLab origin, not external
+assets or redirects; ensure linked hosts are authorized scan targets.
+`--max-artifact-size` limits downloaded bytes, not extracted archive size;
+release scans default to a two-minute `--hit-timeout`.
+
 ## Secret Detection in Pipelines And Artifacts
 
 Nowadays most repositories make use of CI/CD pipelines. A config file per repository `.gitlab-ci.yml` defines what jobs are executed.

docs/introduction/configuration.md

@@ -67,6 +67,9 @@ To view a full example of the available keys run `pipeleek config gen`.
 
 Scan commands inherit from `common`:
 
+The default hit-detection timeout is one minute, or two minutes for `gl release scan`.
+An explicit `common.hit_timeout`, `PIPELEEK_COMMON_HIT_TIMEOUT`, or `--hit-timeout` overrides that default.
+
 ```yaml
 common:
   threads: 2

internal/cmd/gitlab/gitlab.go

@@ -5,6 +5,7 @@ import (
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/container"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/enum"
 	jobtoken "github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/jobToken"
+	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/release"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/renovate"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/runners"
 	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/scan"
@@ -45,6 +46,7 @@ For SOCKS5 proxy:
 	}
 
 	glCmd.AddCommand(scan.NewScanCmd())
+	glCmd.AddCommand(release.NewReleaseRootCmd())
 	glCmd.AddCommand(runners.NewRunnersRootCmd())
 	glCmd.AddCommand(vuln.NewVulnCmd())
 	glCmd.AddCommand(variables.NewVariablesCmd())

internal/cmd/gitlab/gitlab_test.go

@@ -23,6 +23,10 @@ func TestNewGitLabRootCmd(t *testing.T) {
 	assert.Equal(t, "gl [command]", cmd.Use)
 	assert.NotEmpty(t, cmd.Short, "Short description should not be empty")
 	assert.NotEmpty(t, cmd.Long, "Long description should not be empty")
+	releaseScan, remaining, err := cmd.Find([]string{"release", "scan"})
+	require.NoError(t, err)
+	require.Empty(t, remaining)
+	assert.Equal(t, "gl release scan", releaseScan.CommandPath())
 	assert.Equal(t, "GitLab", cmd.GroupID)
 	assert.GreaterOrEqual(t, len(cmd.Commands()), 8,
 		"should have at least 8 subcommands")

internal/cmd/gitlab/release/release.go

@@ -0,0 +1,12 @@
+package release
+
+import (
+	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/release/scan"
+	"github.com/spf13/cobra"
+)
+
+func NewReleaseRootCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "release", Short: "GitLab release related commands"}
+	cmd.AddCommand(scan.NewScanCmd())
+	return cmd
+}

internal/cmd/gitlab/release/release_test.go

@@ -0,0 +1,17 @@
+package release
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewReleaseRootCmd(t *testing.T) {
+	cmd := NewReleaseRootCmd()
+	scan, args, err := cmd.Find([]string{"scan"})
+	require.NoError(t, err)
+	require.Empty(t, args)
+	assert.Equal(t, "scan", scan.Name())
+	assert.NotNil(t, scan.Run)
+}

internal/cmd/gitlab/release/scan/scan.go

@@ -0,0 +1,117 @@
+package scan
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/CompassSecurity/pipeleek/internal/cmd/flags"
+	"github.com/CompassSecurity/pipeleek/pkg/config"
+	releasescan "github.com/CompassSecurity/pipeleek/pkg/gitlab/release/scan"
+	"github.com/CompassSecurity/pipeleek/pkg/logging"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+)
+
+var flagBindings = map[string]string{
+	"url": "gitlab.url", "token": "gitlab.token",
+	"repo": "gitlab.release.scan.repo", "namespace": "gitlab.release.scan.namespace",
+	"search": "gitlab.release.scan.search", "owned": "gitlab.release.scan.owned",
+	"member": "gitlab.release.scan.member", "tag": "gitlab.release.scan.tag",
+	"release-limit": "gitlab.release.scan.release_limit", "include-sources": "gitlab.release.scan.include_sources",
+	"max-artifact-size": "common.max_artifact_size", "threads": "common.threads",
+	"secrets-verification": "common.secrets_verification", "confidence": "common.confidence_filter",
+	"hit-timeout": "common.hit_timeout", "webui": "common.webui",
+}
+
+func NewScanCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "scan", Short: "Scan GitLab release assets for secrets",
+		Long: `Scan linked assets of published GitLab releases for secrets.
+By default, scan all releases of all token-visible projects, including HTTP(S)
+assets hosted outside GitLab. Downloads send the token only to the configured
+GitLab origin; external assets and redirects do not receive it.
+Use --include-sources to also scan one generated source ZIP per release.
+
+Assets exceeding --max-artifact-size (downloaded bytes) are skipped.
+Extracted archive sizes are not limited.
+Other coverage failures are reported while scanning continues, with a nonzero exit.
+--member applies to global project discovery; --owned and --search also apply
+within --namespace, including subgroups. --repo takes precedence over list filters.`,
+		Example: `# Scan all visible projects' release assets
+pipeleek gl release scan --url https://gitlab.example.com --token [redacted]
+
+# Scan one release, including its generated source ZIP
+pipeleek gl release scan --url https://gitlab.example.com --token [redacted] --repo group/project --tag v1.0 --include-sources
+
+# Scan the newest five published releases per project in a namespace
+pipeleek gl release scan --url https://gitlab.example.com --token [redacted] --namespace group --release-limit 5`,
+		Args: cobra.NoArgs, Run: Scan,
+	}
+	var common config.CommonScanOptions
+	flags.AddCommonScanFlagsNoArtifacts(cmd, &common)
+	common.HitTimeout = 2 * time.Minute
+	cmd.Flags().Lookup("hit-timeout").DefValue = common.HitTimeout.String()
+	cmd.Flags().String("max-artifact-size", "500Mb", "Maximum downloaded bytes per asset; extracted archive sizes are not limited")
+	cmd.Flags().StringP("repo", "r", "", "Single repository to scan, format: namespace/repo")
+	cmd.Flags().StringP("namespace", "n", "", "Namespace to scan, including subgroup projects")
+	cmd.Flags().StringP("search", "s", "", "Query string for searching projects")
+	cmd.Flags().BoolP("owned", "o", false, "Scan only user owned repositories")
+	cmd.Flags().BoolP("member", "m", false, "Scan projects the user is a member of (global project discovery)")
+	cmd.Flags().String("tag", "", "Exact release tag to scan (requires --repo)")
+	cmd.Flags().Int("release-limit", 0, "Maximum published releases per project, newest first; 0 scans all")
+	cmd.Flags().Bool("include-sources", false, "Also scan one generated source ZIP per release")
+	return cmd
+}
+
+func optionsFromConfig() (releasescan.Options, error) {
+	config.GetViper().SetDefault("common.hit_timeout", "120s")
+	maxBytes, err := config.ParseMaxArtifactSize(config.GetString("common.max_artifact_size"))
+	if err != nil {
+		return releasescan.Options{}, err
+	}
+	rawTimeout := config.GetString("common.hit_timeout")
+	if _, err := strconv.ParseInt(rawTimeout, 10, 64); err == nil {
+		rawTimeout += "s"
+	}
+	timeout, err := time.ParseDuration(rawTimeout)
+	if err != nil {
+		return releasescan.Options{}, fmt.Errorf("invalid hit-timeout: %w", err)
+	}
+	o := releasescan.Options{
+		URL: config.GetString("gitlab.url"), Token: config.GetString("gitlab.token"),
+		Repo: config.GetString("gitlab.release.scan.repo"), Namespace: config.GetString("gitlab.release.scan.namespace"),
+		Search: config.GetString("gitlab.release.scan.search"), Owned: config.GetBool("gitlab.release.scan.owned"),
+		Member: config.GetBool("gitlab.release.scan.member"), Tag: config.GetString("gitlab.release.scan.tag"),
+		ReleaseLimit:   config.GetInt("gitlab.release.scan.release_limit"),
+		IncludeSources: config.GetBool("gitlab.release.scan.include_sources"),
+		MaxBytes:       maxBytes, Threads: config.GetInt("common.threads"),
+		SecretsVerification: config.GetBool("common.secrets_verification"),
+		Confidence:          config.GetStringSlice("common.confidence_filter"), HitTimeout: timeout,
+	}
+	return o, o.Validate()
+}
+
+func Scan(cmd *cobra.Command, args []string) {
+	config.NewCommandSetup(cmd).WithFlagBindings(flagBindings).
+		RequireKeys("gitlab.url", "gitlab.token").MustBind()
+	opts, err := optionsFromConfig()
+	if err != nil {
+		log.Fatal().Err(err).Msg("Invalid release scan configuration")
+	}
+	ui := flags.StartScanWebUI(cmd, opts.URL, flagBindings)
+	if ui != nil {
+		defer ui.Close()
+	}
+	s := releasescan.NewScanner(opts)
+	logging.RegisterStatusHook(s.Status)
+	if err := s.Scan(); err != nil {
+		if ui != nil {
+			_ = ui.Close()
+		}
+		log.Fatal().Err(err).Msg("Release scan failed")
+	}
+	if ui != nil {
+		ui.Wait()
+	}
+}

internal/cmd/gitlab/release/scan/scan_test.go

@@ -0,0 +1,122 @@
+package scan
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/CompassSecurity/pipeleek/internal/cmd/testutil"
+	"github.com/CompassSecurity/pipeleek/pkg/config"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewScanCmd(t *testing.T) {
+	cmd := NewScanCmd()
+	testutil.AssertAllFlagsHaveBindings(t, cmd, flagBindings, "url", "token")
+	assert.Equal(t, reflect.ValueOf(Scan).Pointer(), reflect.ValueOf(cmd.Run).Pointer())
+	assert.Error(t, cmd.Args(cmd, []string{"extra"}))
+	for name, value := range map[string]string{
+		"release-limit": "0", "include-sources": "false", "max-artifact-size": "500Mb",
+		"threads": "4", "hit-timeout": "2m0s", "secrets-verification": "true",
+	} {
+		require.NotNil(t, cmd.Flags().Lookup(name))
+		assert.Equal(t, value, cmd.Flags().Lookup(name).DefValue, name)
+	}
+	assert.Nil(t, cmd.Flags().Lookup("artifacts"))
+	assert.Nil(t, cmd.Flags().Lookup("cookie"))
+}
+
+func TestOptionsFromConfig(t *testing.T) {
+	t.Setenv("PIPELEEK_NO_CONFIG", "1")
+	t.Setenv("PIPELEEK_GITLAB_RELEASE_SCAN_TAG", "v2")
+	t.Setenv("PIPELEEK_GITLAB_RELEASE_SCAN_REPO", "group/project")
+	require.NoError(t, config.InitializeViper(""))
+	parent := &cobra.Command{Use: "gl"}
+	parent.PersistentFlags().String("url", "", "")
+	parent.PersistentFlags().String("token", "", "")
+	cmd := NewScanCmd()
+	parent.AddCommand(cmd)
+	require.NoError(t, cmd.ParseFlags([]string{"--url", "https://gitlab.example.com", "--token", "test-token", "--tag", "v3"}))
+	require.NoError(t, config.NewCommandSetup(cmd).WithFlagBindings(flagBindings).Bind())
+	opts, err := optionsFromConfig()
+	require.NoError(t, err)
+	assert.Equal(t, 2*time.Minute, opts.HitTimeout)
+	config.GetViper().Set("common.hit_timeout", "120")
+	opts, err = optionsFromConfig()
+	require.NoError(t, err)
+	assert.Equal(t, "v3", opts.Tag)
+	assert.Equal(t, "group/project", opts.Repo)
+	assert.Equal(t, 120*time.Second, opts.HitTimeout)
+	assert.Equal(t, int64(500000000), opts.MaxBytes)
+	config.GetViper().Set("common.max_artifact_size", "bad")
+	_, err = optionsFromConfig()
+	require.Error(t, err)
+	config.GetViper().Set("common.max_artifact_size", "1Mb")
+	config.GetViper().Set("common.hit_timeout", "bad")
+	_, err = optionsFromConfig()
+	require.ErrorContains(t, err, "hit-timeout")
+}
+
+func TestConfigurationPrecedence(t *testing.T) {
+	t.Setenv("PIPELEEK_NO_CONFIG", "")
+	file := filepath.Join(t.TempDir(), "pipeleek.yaml")
+	require.NoError(t, os.WriteFile(file, []byte(`gitlab:
+  url: https://gitlab.example.com
+  token: test-token
+  release:
+    scan:
+      repo: group/project
+      release_limit: 7
+      include_sources: true
+common:
+  threads: 3
+`), 0600))
+	t.Setenv("PIPELEEK_GITLAB_RELEASE_SCAN_RELEASE_LIMIT", "5")
+	require.NoError(t, config.InitializeViper(file))
+	cmd := NewScanCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--release-limit", "2"}))
+	require.NoError(t, config.NewCommandSetup(cmd).WithFlagBindings(flagBindings).Bind())
+	opts, err := optionsFromConfig()
+	require.NoError(t, err)
+	assert.Equal(t, 2, opts.ReleaseLimit)
+	assert.Equal(t, 3, opts.Threads)
+	assert.True(t, opts.IncludeSources)
+	assert.Equal(t, "group/project", opts.Repo)
+	require.NoError(t, config.InitializeViper(file))
+	cmd = NewScanCmd()
+	require.NoError(t, config.NewCommandSetup(cmd).WithFlagBindings(flagBindings).Bind())
+	opts, err = optionsFromConfig()
+	require.NoError(t, err)
+	assert.Equal(t, 5, opts.ReleaseLimit)
+}
+
+func TestHitTimeoutOverrides(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		env  string
+		args []string
+		want time.Duration
+	}{
+		{"default", "", nil, 2 * time.Minute},
+		{"environment", "45s", nil, 45 * time.Second},
+		{"flag", "45s", []string{"--hit-timeout", "30s"}, 30 * time.Second},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Setenv("PIPELEEK_NO_CONFIG", "1")
+			t.Setenv("PIPELEEK_GITLAB_URL", "https://gitlab.example.com")
+			t.Setenv("PIPELEEK_GITLAB_TOKEN", "test-token")
+			t.Setenv("PIPELEEK_COMMON_HIT_TIMEOUT", tc.env)
+			require.NoError(t, config.InitializeViper(""))
+			cmd := NewScanCmd()
+			require.NoError(t, cmd.ParseFlags(tc.args))
+			require.NoError(t, config.NewCommandSetup(cmd).WithFlagBindings(flagBindings).Bind())
+			opts, err := optionsFromConfig()
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, opts.HitTimeout)
+		})
+	}
+}

pkg/config/gen/release_test.go

@@ -0,0 +1,43 @@
+package gen_test
+
+import (
+	"testing"
+
+	"github.com/CompassSecurity/pipeleek/internal/cmd/gitlab/release"
+	"github.com/CompassSecurity/pipeleek/pkg/config/gen"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestReleaseScanConfig(t *testing.T) {
+	root := &cobra.Command{Use: "pipeleek"}
+	gl := &cobra.Command{Use: "gl"}
+	gl.PersistentFlags().String("url", "", "")
+	gl.PersistentFlags().String("token", "", "")
+	gl.AddCommand(release.NewReleaseRootCmd())
+	root.AddCommand(gl)
+	for _, name := range []string{"repo", "namespace", "search", "owned", "member", "tag", "release_limit", "include_sources"} {
+		assert.True(t, gen.IsAllowedConfigPath(root, "gitlab.release.scan."+name), name)
+	}
+	for _, name := range []string{"threads", "max_artifact_size", "hit_timeout", "webui", "secrets_verification"} {
+		assert.True(t, gen.IsAllowedConfigPath(root, "common."+name), name)
+	}
+	content := gen.GenerateExampleConfig(root)
+	var parsed struct {
+		GitLab struct {
+			Release struct {
+				Scan struct {
+					ReleaseLimit   int  `yaml:"release_limit"`
+					IncludeSources bool `yaml:"include_sources"`
+				}
+			}
+		}
+	}
+	require.NoError(t, yaml.Unmarshal([]byte(content), &parsed))
+	assert.Zero(t, parsed.GitLab.Release.Scan.ReleaseLimit)
+	assert.False(t, parsed.GitLab.Release.Scan.IncludeSources)
+	assert.Contains(t, content, "PIPELEEK_GITLAB_RELEASE_SCAN_RELEASE_LIMIT")
+	assert.Contains(t, content, `hit_timeout: "2m0s"`)
+}