From e00390e3937d6f2b6f6deda3097aea7fe27052f0 Mon Sep 17 00:00:00 2001 From: ThryLox Date: Tue, 30 Jun 2026 01:18:54 -0400 Subject: [PATCH 1/2] feat: add csv output support with customizable fields Fixes #745. Introduces a new -csv / -c flag to output results in CSV format, supporting customizable fields aligned with the -field flag (ip, port, host, url). Defaults to 'ip,port,host' if not overridden. Added unit tests for CSV serialization in the runner package. --- runner/options.go | 6 ++++ runner/output_writer.go | 61 ++++++++++++++++++++++++++++++++++++ runner/runner.go | 11 ++++++- runner/runner_test.go | 68 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 runner/runner_test.go diff --git a/runner/options.go b/runner/options.go index c09a1d9e..130beb7f 100644 --- a/runner/options.go +++ b/runner/options.go @@ -37,6 +37,7 @@ type Options struct { OutputFields string JSON bool Raw bool + CSV bool Limit int Silent bool Verbose bool @@ -122,6 +123,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFields, "field", "f", "ip:port", "field to display in output (ip,port,host)"), flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.Raw, "raw", "r", false, "write raw output as received by the remote api"), + flagSet.BoolVarP(&options.CSV, "csv", "c", false, "write output in CSV format"), flagSet.IntVarP(&options.Limit, "limit", "l", 100, "limit the number of results to return"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in output"), ) @@ -201,6 +203,7 @@ func ParseOptions() *Options { // Validate the options passed by the user and if any // invalid options have been used, exit. + options.configureOutput() if err := options.validateOptions(); err != nil { gologger.Fatal().Msgf("Program exiting: %s\n", err) } @@ -220,6 +223,9 @@ func (options *Options) configureOutput() { if options.Silent { gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } + if options.CSV && options.OutputFields == "ip:port" { + options.OutputFields = "ip,port,host" + } } func (options *Options) loadConfigFrom(location string) error { diff --git a/runner/output_writer.go b/runner/output_writer.go index 0277d864..d53f0b14 100644 --- a/runner/output_writer.go +++ b/runner/output_writer.go @@ -1,11 +1,15 @@ package runner import ( + "bytes" "crypto/sha1" + "encoding/csv" "fmt" "io" "os" + "strings" "sync" + "unicode" lru "github.com/hashicorp/golang-lru" "github.com/projectdiscovery/uncover/sources" @@ -69,6 +73,63 @@ func (o *OutputWriter) WriteJsonData(data sources.Result) { o.Write([]byte(data.JSON())) } +func (o *OutputWriter) WriteCSVRow(row []string) { + o.Lock() + defer o.Unlock() + + var buf bytes.Buffer + w := csv.NewWriter(&buf) + if err := w.Write(row); err != nil { + return + } + w.Flush() + + for _, writer := range o.writers { + _, _ = writer.Write(buf.Bytes()) + } +} + +func (o *OutputWriter) WriteCSVData(data sources.Result, fields []string) { + values := getFieldValues(data, fields) + dupKey := strings.Join(values, ",") + if o.findDuplicate(dupKey, true) { + return + } + o.WriteCSVRow(values) +} + +func parseFields(fields string) []string { + var parsed []string + for _, f := range strings.FieldsFunc(fields, func(r rune) bool { + return r == ',' || r == ':' || r == ';' || unicode.IsSpace(r) + }) { + f = strings.TrimSpace(f) + if f != "" { + parsed = append(parsed, f) + } + } + return parsed +} + +func getFieldValues(result sources.Result, fields []string) []string { + values := make([]string, len(fields)) + for i, f := range fields { + switch strings.ToLower(f) { + case "ip": + values[i] = result.IP + case "port": + values[i] = fmt.Sprint(result.Port) + case "host": + values[i] = result.Host + case "url": + values[i] = result.Url + default: + values[i] = "" + } + } + return values +} + // Close closes the output writers func (o *OutputWriter) Close() { // Iterate over the writers and close the file writers diff --git a/runner/runner.go b/runner/runner.go index 01285705..3bce0d14 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -58,8 +58,14 @@ func NewRunner(options *Options) (*Runner, error) { return runner, nil } -// RunEnumeration runs the subdomain enumeration flow on the targets specified +// Run runs the subdomain enumeration flow on the targets specified func (r *Runner) Run(ctx context.Context) error { + var csvFields []string + if r.options.CSV { + csvFields = parseFields(r.options.OutputFields) + r.outputWriter.WriteCSVRow(csvFields) + } + resultCallback := func(result sources.Result) { optionFields := r.options.OutputFields switch { @@ -71,6 +77,9 @@ func (r *Runner) Run(ctx context.Context) error { case r.options.Raw: gologger.Verbose().Label(result.Source).Msgf("%s\n", result.RawData()) r.outputWriter.WriteString(result.RawData()) + case r.options.CSV: + gologger.Verbose().Label(result.Source).Msgf("%s\n", strings.Join(getFieldValues(result, csvFields), ",")) + r.outputWriter.WriteCSVData(result, csvFields) default: port := fmt.Sprint(result.Port) replacer := strings.NewReplacer( diff --git a/runner/runner_test.go b/runner/runner_test.go new file mode 100644 index 00000000..5cb1ea13 --- /dev/null +++ b/runner/runner_test.go @@ -0,0 +1,68 @@ +package runner + +import ( + "bytes" + "strings" + "testing" + + "github.com/projectdiscovery/uncover/sources" +) + +func TestOutputWriter_WriteCSVData(t *testing.T) { + writer, err := NewOutputWriter() + if err != nil { + t.Fatalf("Failed to create OutputWriter: %s", err) + } + + var buf bytes.Buffer + writer.AddWriters(&buf) + + fields := []string{"ip", "port", "host"} + writer.WriteCSVRow(fields) + + result := sources.Result{ + IP: "192.168.1.1", + Port: 80, + Host: "localhost", + } + + writer.WriteCSVData(result, fields) + + output := buf.String() + expectedHeader := "ip,port,host\n" + expectedRow := "192.168.1.1,80,localhost\n" + + if !strings.Contains(output, expectedHeader) { + t.Errorf("Expected output to contain header %q, got %q", expectedHeader, output) + } + if !strings.Contains(output, expectedRow) { + t.Errorf("Expected output to contain row %q, got %q", expectedRow, output) + } +} + +func TestParseFields(t *testing.T) { + tests := []struct { + input string + expected []string + }{ + {"ip,port,host", []string{"ip", "port", "host"}}, + {"ip:port:host", []string{"ip", "port", "host"}}, + {"ip;port;host", []string{"ip", "port", "host"}}, + {"ip port host", []string{"ip", "port", "host"}}, + {"ip\tport\thost", []string{"ip", "port", "host"}}, + {"ip\nport\nhost", []string{"ip", "port", "host"}}, + } + + for _, tt := range tests { + actual := parseFields(tt.input) + if len(actual) != len(tt.expected) { + t.Errorf("For %q expected length %d, got %d", tt.input, len(tt.expected), len(actual)) + continue + } + for i, v := range actual { + if v != tt.expected[i] { + t.Errorf("For %q expected index %d to be %q, got %q", tt.input, i, tt.expected[i], v) + } + } + } +} From ca3a7fc0f8d437687ad3bc03d0f2319ebc0545b2 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 2 Jul 2026 23:17:37 +0200 Subject: [PATCH 2/2] address review --- README.md | 1 + runner/options.go | 11 ++++++++++ runner/runner.go | 11 +++++++--- runner/runner_test.go | 48 +++++++++++++++++++++++++++++++++++-------- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d5e8bbf6..f6a80d2e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ OUTPUT: -f, -field string field to display in output (ip,port,host) (default "ip:port") -j, -json write output in JSONL(ines) format -r, -raw write raw output as received by the remote api + -c, -csv write output in CSV format (defaults to ip,port,host fields) -l, -limit int limit the number of results to return (default 100) -nc, -no-color disable colors in output diff --git a/runner/options.go b/runner/options.go index 130beb7f..7970a9ce 100644 --- a/runner/options.go +++ b/runner/options.go @@ -268,6 +268,17 @@ func (options *Options) validateOptions() error { return errors.New("both verbose and silent mode specified") } + // Only one output format may be selected at a time. + formats := 0 + for _, enabled := range []bool{options.JSON, options.Raw, options.CSV} { + if enabled { + formats++ + } + } + if formats > 1 { + return errors.New("only one of -json, -raw, -csv can be used at a time") + } + // Validate threads and options if genericutil.EqualsAll(0, len(options.Engine), diff --git a/runner/runner.go b/runner/runner.go index 3bce0d14..ad040b88 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -60,8 +60,11 @@ func NewRunner(options *Options) (*Runner, error) { // Run runs the subdomain enumeration flow on the targets specified func (r *Runner) Run(ctx context.Context) error { + // CSV is only the effective format when no higher-precedence output mode + // (JSON/Raw) is set; otherwise skip the header so we don't emit a stray row. + csvOutput := r.options.CSV && !r.options.JSON && !r.options.Raw var csvFields []string - if r.options.CSV { + if csvOutput { csvFields = parseFields(r.options.OutputFields) r.outputWriter.WriteCSVRow(csvFields) } @@ -77,8 +80,10 @@ func (r *Runner) Run(ctx context.Context) error { case r.options.Raw: gologger.Verbose().Label(result.Source).Msgf("%s\n", result.RawData()) r.outputWriter.WriteString(result.RawData()) - case r.options.CSV: - gologger.Verbose().Label(result.Source).Msgf("%s\n", strings.Join(getFieldValues(result, csvFields), ",")) + case csvOutput: + if r.options.Verbose { + gologger.Verbose().Label(result.Source).Msgf("%s\n", strings.Join(getFieldValues(result, csvFields), ",")) + } r.outputWriter.WriteCSVData(result, csvFields) default: port := fmt.Sprint(result.Port) diff --git a/runner/runner_test.go b/runner/runner_test.go index 5cb1ea13..57a712fb 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -2,7 +2,6 @@ package runner import ( "bytes" - "strings" "testing" "github.com/projectdiscovery/uncover/sources" @@ -20,23 +19,54 @@ func TestOutputWriter_WriteCSVData(t *testing.T) { fields := []string{"ip", "port", "host"} writer.WriteCSVRow(fields) + // Host contains a comma and quotes to exercise RFC-4180 escaping. result := sources.Result{ IP: "192.168.1.1", Port: 80, - Host: "localhost", + Host: `local,"host"`, } + writer.WriteCSVData(result, fields) + // A second identical write must be suppressed as a duplicate. writer.WriteCSVData(result, fields) - output := buf.String() - expectedHeader := "ip,port,host\n" - expectedRow := "192.168.1.1,80,localhost\n" + expected := "ip,port,host\n192.168.1.1,80,\"local,\"\"host\"\"\"\n" + if got := buf.String(); got != expected { + t.Fatalf("unexpected CSV output:\n got: %q\nwant: %q", got, expected) + } +} - if !strings.Contains(output, expectedHeader) { - t.Errorf("Expected output to contain header %q, got %q", expectedHeader, output) +func TestOutputWriter_WriteCSVData_DistinctProjections(t *testing.T) { + writer, err := NewOutputWriter() + if err != nil { + t.Fatalf("Failed to create OutputWriter: %s", err) } - if !strings.Contains(output, expectedRow) { - t.Errorf("Expected output to contain row %q, got %q", expectedRow, output) + + var buf bytes.Buffer + writer.AddWriters(&buf) + + // Same IP but different hosts must produce two rows when host is projected. + writer.WriteCSVData(sources.Result{IP: "1.1.1.1", Port: 80, Host: "a.example.com"}, []string{"ip", "port", "host"}) + writer.WriteCSVData(sources.Result{IP: "1.1.1.1", Port: 80, Host: "b.example.com"}, []string{"ip", "port", "host"}) + + expected := "1.1.1.1,80,a.example.com\n1.1.1.1,80,b.example.com\n" + if got := buf.String(); got != expected { + t.Fatalf("distinct rows were incorrectly deduplicated:\n got: %q\nwant: %q", got, expected) + } +} + +func TestGetFieldValues(t *testing.T) { + result := sources.Result{IP: "1.1.1.1", Port: 443, Host: "example.com", Url: "https://example.com"} + got := getFieldValues(result, []string{"IP", "port", "host", "url", "unknown"}) + want := []string{"1.1.1.1", "443", "example.com", "https://example.com", ""} + + if len(got) != len(want) { + t.Fatalf("expected %d values, got %d (%v)", len(want), len(got), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("field %d: expected %q, got %q", i, want[i], got[i]) + } } }