feat: add csv output support with customizable fields (#745)

README.md

@@ -113,6 +113,7 @@ OUTPUT:
    -f, -field string   field to display in output (ip,port,host) (default "ip:port")
    -j, -json           write output in JSONL(ines) format
    -r, -raw            write raw output as received by the remote api
+   -c, -csv            write output in CSV format (defaults to ip,port,host fields)
    -l, -limit int      limit the number of results to return (default 100)
    -nc, -no-color      disable colors in output
 

runner/options.go

@@ -37,6 +37,7 @@ type Options struct {
 	OutputFields         string
 	JSON                 bool
 	Raw                  bool
+	CSV                  bool
 	Limit                int
 	Silent               bool
 	Verbose              bool
@@ -122,6 +123,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.OutputFields, "field", "f", "ip:port", "field to display in output (ip,port,host)"),
 		flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.Raw, "raw", "r", false, "write raw output as received by the remote api"),
+		flagSet.BoolVarP(&options.CSV, "csv", "c", false, "write output in CSV format"),
 		flagSet.IntVarP(&options.Limit, "limit", "l", 100, "limit the number of results to return"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in output"),
 	)
@@ -201,6 +203,7 @@ func ParseOptions() *Options {
 
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
+	options.configureOutput()
 	if err := options.validateOptions(); err != nil {
 		gologger.Fatal().Msgf("Program exiting: %s\n", err)
 	}
@@ -220,6 +223,9 @@ func (options *Options) configureOutput() {
 	if options.Silent {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
 	}
+	if options.CSV && options.OutputFields == "ip:port" {
+		options.OutputFields = "ip,port,host"
+	}
 }
 
 func (options *Options) loadConfigFrom(location string) error {
@@ -262,6 +268,17 @@ func (options *Options) validateOptions() error {
 		return errors.New("both verbose and silent mode specified")
 	}
 
+	// Only one output format may be selected at a time.
+	formats := 0
+	for _, enabled := range []bool{options.JSON, options.Raw, options.CSV} {
+		if enabled {
+			formats++
+		}
+	}
+	if formats > 1 {
+		return errors.New("only one of -json, -raw, -csv can be used at a time")
+	}
+
 	// Validate threads and options
 	if genericutil.EqualsAll(0,
 		len(options.Engine),

runner/output_writer.go

@@ -1,11 +1,15 @@
 package runner
 
 import (
+	"bytes"
 	"crypto/sha1"
+	"encoding/csv"
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
+	"unicode"
 
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/projectdiscovery/uncover/sources"
@@ -69,6 +73,63 @@ func (o *OutputWriter) WriteJsonData(data sources.Result) {
 	o.Write([]byte(data.JSON()))
 }
 
+func (o *OutputWriter) WriteCSVRow(row []string) {
+	o.Lock()
+	defer o.Unlock()
+
+	var buf bytes.Buffer
+	w := csv.NewWriter(&buf)
+	if err := w.Write(row); err != nil {
+		return
+	}
+	w.Flush()
+
+	for _, writer := range o.writers {
+		_, _ = writer.Write(buf.Bytes())
+	}
+}
+
+func (o *OutputWriter) WriteCSVData(data sources.Result, fields []string) {
+	values := getFieldValues(data, fields)
+	dupKey := strings.Join(values, ",")
+	if o.findDuplicate(dupKey, true) {
+		return
+	}
+	o.WriteCSVRow(values)
+}
+
+func parseFields(fields string) []string {
+	var parsed []string
+	for _, f := range strings.FieldsFunc(fields, func(r rune) bool {
+		return r == ',' || r == ':' || r == ';' || unicode.IsSpace(r)
+	}) {
+		f = strings.TrimSpace(f)
+		if f != "" {
+			parsed = append(parsed, f)
+		}
+	}
+	return parsed
+}
+
+func getFieldValues(result sources.Result, fields []string) []string {
+	values := make([]string, len(fields))
+	for i, f := range fields {
+		switch strings.ToLower(f) {
+		case "ip":
+			values[i] = result.IP
+		case "port":
+			values[i] = fmt.Sprint(result.Port)
+		case "host":
+			values[i] = result.Host
+		case "url":
+			values[i] = result.Url
+		default:
+			values[i] = ""
+		}
+	}
+	return values
+}
+
 // Close closes the output writers
 func (o *OutputWriter) Close() {
 	// Iterate over the writers and close the file writers

runner/runner.go

@@ -58,8 +58,17 @@ func NewRunner(options *Options) (*Runner, error) {
 	return runner, nil
 }
 
-// RunEnumeration runs the subdomain enumeration flow on the targets specified
+// Run runs the subdomain enumeration flow on the targets specified
 func (r *Runner) Run(ctx context.Context) error {
+	// CSV is only the effective format when no higher-precedence output mode
+	// (JSON/Raw) is set; otherwise skip the header so we don't emit a stray row.
+	csvOutput := r.options.CSV && !r.options.JSON && !r.options.Raw
+	var csvFields []string
+	if csvOutput {
+		csvFields = parseFields(r.options.OutputFields)
+		r.outputWriter.WriteCSVRow(csvFields)
+	}
+
 	resultCallback := func(result sources.Result) {
 		optionFields := r.options.OutputFields
 		switch {
@@ -71,6 +80,11 @@ func (r *Runner) Run(ctx context.Context) error {
 		case r.options.Raw:
 			gologger.Verbose().Label(result.Source).Msgf("%s\n", result.RawData())
 			r.outputWriter.WriteString(result.RawData())
+		case csvOutput:
+			if r.options.Verbose {
+				gologger.Verbose().Label(result.Source).Msgf("%s\n", strings.Join(getFieldValues(result, csvFields), ","))
+			}
+			r.outputWriter.WriteCSVData(result, csvFields)
 		default:
 			port := fmt.Sprint(result.Port)
 			replacer := strings.NewReplacer(

runner/runner_test.go

@@ -0,0 +1,98 @@
+package runner
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/projectdiscovery/uncover/sources"
+)
+
+func TestOutputWriter_WriteCSVData(t *testing.T) {
+	writer, err := NewOutputWriter()
+	if err != nil {
+		t.Fatalf("Failed to create OutputWriter: %s", err)
+	}
+
+	var buf bytes.Buffer
+	writer.AddWriters(&buf)
+
+	fields := []string{"ip", "port", "host"}
+	writer.WriteCSVRow(fields)
+
+	// Host contains a comma and quotes to exercise RFC-4180 escaping.
+	result := sources.Result{
+		IP:   "192.168.1.1",
+		Port: 80,
+		Host: `local,"host"`,
+	}
+
+	writer.WriteCSVData(result, fields)
+	// A second identical write must be suppressed as a duplicate.
+	writer.WriteCSVData(result, fields)
+
+	expected := "ip,port,host\n192.168.1.1,80,\"local,\"\"host\"\"\"\n"
+	if got := buf.String(); got != expected {
+		t.Fatalf("unexpected CSV output:\n got: %q\nwant: %q", got, expected)
+	}
+}
+
+func TestOutputWriter_WriteCSVData_DistinctProjections(t *testing.T) {
+	writer, err := NewOutputWriter()
+	if err != nil {
+		t.Fatalf("Failed to create OutputWriter: %s", err)
+	}
+
+	var buf bytes.Buffer
+	writer.AddWriters(&buf)
+
+	// Same IP but different hosts must produce two rows when host is projected.
+	writer.WriteCSVData(sources.Result{IP: "1.1.1.1", Port: 80, Host: "a.example.com"}, []string{"ip", "port", "host"})
+	writer.WriteCSVData(sources.Result{IP: "1.1.1.1", Port: 80, Host: "b.example.com"}, []string{"ip", "port", "host"})
+
+	expected := "1.1.1.1,80,a.example.com\n1.1.1.1,80,b.example.com\n"
+	if got := buf.String(); got != expected {
+		t.Fatalf("distinct rows were incorrectly deduplicated:\n got: %q\nwant: %q", got, expected)
+	}
+}
+
+func TestGetFieldValues(t *testing.T) {
+	result := sources.Result{IP: "1.1.1.1", Port: 443, Host: "example.com", Url: "https://example.com"}
+	got := getFieldValues(result, []string{"IP", "port", "host", "url", "unknown"})
+	want := []string{"1.1.1.1", "443", "example.com", "https://example.com", ""}
+
+	if len(got) != len(want) {
+		t.Fatalf("expected %d values, got %d (%v)", len(want), len(got), got)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Errorf("field %d: expected %q, got %q", i, want[i], got[i])
+		}
+	}
+}
+
+func TestParseFields(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected []string
+	}{
+		{"ip,port,host", []string{"ip", "port", "host"}},
+		{"ip:port:host", []string{"ip", "port", "host"}},
+		{"ip;port;host", []string{"ip", "port", "host"}},
+		{"ip port host", []string{"ip", "port", "host"}},
+		{"ip\tport\thost", []string{"ip", "port", "host"}},
+		{"ip\nport\nhost", []string{"ip", "port", "host"}},
+	}
+
+	for _, tt := range tests {
+		actual := parseFields(tt.input)
+		if len(actual) != len(tt.expected) {
+			t.Errorf("For %q expected length %d, got %d", tt.input, len(tt.expected), len(actual))
+			continue
+		}
+		for i, v := range actual {
+			if v != tt.expected[i] {
+				t.Errorf("For %q expected index %d to be %q, got %q", tt.input, i, tt.expected[i], v)
+			}
+		}
+	}
+}