Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
- **[Google](https://www.google.com)**
- **[Onyphe](https://www.onyphe.io/)**
- **[Driftnet](https://driftnet.io)**
- **[DayDayMap](https://www.daydaymap.com)**
- **[NerdyData](https://www.nerdydata.com/?utm_source=projectdiscovery/uncover)**
- Multiple API key input support
- Automatic API key randomization
Expand Down Expand Up @@ -78,7 +79,7 @@ Usage:
Flags:
INPUT:
-q, -query string[] search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
-e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet) (default shodan)
-e, -engine string[] search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google,driftnet,daydaymap) (default shodan)
-asq, -awesome-search-queries string[] use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')

SEARCH-ENGINE:
Expand All @@ -96,6 +97,7 @@ SEARCH-ENGINE:
-gg, -google string[] search query for google (example: -google 'query.txt')
-on, -onyphe string[] search query for onyphe (example: -onyphe 'query.txt')
-df, -driftnet string[] search query for driftnet (example: -driftnet 'query.txt')
-ddm, -daydaymap string[] search query for daydaymap (example: -daydaymap 'query.txt')

CONFIG:
-pc, -provider string provider configuration file (default "$CONFIG/uncover/provider-config.yaml")
Expand Down Expand Up @@ -171,6 +173,9 @@ onyphe:
driftnet:
- DRIFTNET_API_KEY_1
- DRIFTNET_API_KEY_2
daydaymap:
- DAYDAYMAP_API_KEY_1
- DAYDAYMAP_API_KEY_2
nerdydata:
- NERDYDATA_API_KEY_1
- NERDYDATA_API_KEY_2
Expand All @@ -197,10 +202,11 @@ export GOOGLE_API_KEY=xxx
export GOOGLE_API_CX=xxx
export ONYPHE_API_KEY=xxx
export DRIFTNET_API_KEY=xxx
export DAYDAYMAP_API_KEY=xxx
export NERDYDATA_API_KEY=xxx
```

Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.ai), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup), [Driftnet](https://driftnet.io/auth?state=signup) and [NerdyData](https://www.nerdydata.com/api?utm_source=projectdiscovery/uncover).
Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.ai), [Netlas](https://app.netlas.io/registration/), [CriminalIP](https://www.criminalip.io/register), [Publicwww](https://publicwww.com/profile/signup.html), Google [[1]](https://developers.google.com/custom-search/v1/introduction#identify_your_application_to_google_with_api_key),[[2]](https://programmablesearchengine.google.com/controlpanel/create), [Onyphe](https://search.onyphe.io/signup), [Driftnet](https://driftnet.io/auth?state=signup), [DayDayMap](https://www.daydaymap.com) and [NerdyData](https://www.nerdydata.com/api?utm_source=projectdiscovery/uncover).

## Running Uncover

Expand Down Expand Up @@ -279,7 +285,7 @@ uncover -q dorks.txt
**uncover** supports multiple search engine, as default **shodan** is used, `-e` flag can be used to run same query against any or all search engines.

```console
echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye,netlas,criminalip,driftnet
echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye,netlas,criminalip,driftnet,daydaymap

__ ______ _________ _ _____ _____
/ / / / __ \/ ___/ __ \ | / / _ \/ ___/
Expand Down Expand Up @@ -311,7 +317,7 @@ echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye,netlas,criminalip


```console
uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira' -zoomeye 'app:"Atlassian JIRA"' -netlas 'jira' -criminalip 'Jira' -driftnet 'field=product-tag:jira'
uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira' -zoomeye 'app:"Atlassian JIRA"' -netlas 'jira' -criminalip 'Jira' -driftnet 'field=product-tag:jira' -daydaymap 'Jira'

__ ______ _________ _ _____ _____
/ / / / __ \/ ___/ __ \ | / / _ \/ ___/
Expand Down
8 changes: 7 additions & 1 deletion runner/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type Options struct {
Onyphe goflags.StringSlice
Driftnet goflags.StringSlice
GreyNoise goflags.StringSlice
Daydaymap goflags.StringSlice
NerdyData goflags.StringSlice
DisableUpdateCheck bool
}
Expand All @@ -75,7 +76,7 @@ func ParseOptions() *Options {

flagSet.CreateGroup("input", "Input",
flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,nerdydata) (default shodan)", goflags.FileNormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google,odin,binaryedge,onyphe,driftnet,greynoise,daydaymap,nerdydata) (default shodan)", goflags.FileNormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions),
)

Expand All @@ -97,6 +98,7 @@ func ParseOptions() *Options {
flagSet.StringSliceVarP(&options.Onyphe, "onyphe", "on", nil, "search query for onyphe (example: -onyphe 'query.txt')", goflags.FileStringSliceOptions),
flagSet.StringSliceVarP(&options.Driftnet, "driftnet", "df", nil, "search query for driftnet (example: -driftnet 'query.txt')", goflags.FileStringSliceOptions),
flagSet.StringSliceVarP(&options.GreyNoise, "greynoise", "gn", nil, "search query for greynoise (example: -greynoise 'query.txt')", goflags.FileStringSliceOptions),
flagSet.StringSliceVarP(&options.Daydaymap, "daydaymap", "ddm", nil, "search query for daydaymap (example: -daydaymap 'query.txt')", goflags.FileStringSliceOptions),
flagSet.StringSliceVarP(&options.NerdyData, "nerdydata", "nd", nil, "search query for NerdyData (example: -nerdydata 'query.txt')", goflags.FileStringSliceOptions),
)

Expand Down Expand Up @@ -175,6 +177,7 @@ func ParseOptions() *Options {
len(options.Onyphe),
len(options.Driftnet),
len(options.GreyNoise),
len(options.Daydaymap),
len(options.NerdyData)) {
options.Engine = append(options.Engine, "shodan")
}
Expand Down Expand Up @@ -249,6 +252,7 @@ func (options *Options) validateOptions() error {
len(options.Onyphe),
len(options.Driftnet),
len(options.GreyNoise),
len(options.Daydaymap),
len(options.NerdyData)) {
return errors.New("no query provided")
}
Expand Down Expand Up @@ -278,6 +282,7 @@ func (options *Options) validateOptions() error {
len(options.Onyphe),
len(options.Driftnet),
len(options.GreyNoise),
len(options.Daydaymap),
len(options.NerdyData)) {
return errors.New("no engine specified")
}
Expand Down Expand Up @@ -323,6 +328,7 @@ func appendAllQueries(options *Options) {
appendQuery(options, "onyphe", options.Onyphe...)
appendQuery(options, "driftnet", options.Driftnet...)
appendQuery(options, "greynoise", options.GreyNoise...)
appendQuery(options, "daydaymap", options.Daydaymap...)
appendQuery(options, "nerdydata", options.NerdyData...)
}

Expand Down
19 changes: 18 additions & 1 deletion sources/agent.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,28 @@
package sources

import "context"

type Query struct {
Query string
Limit int
}

// Agent fans out a query to one upstream search engine and streams results
// on the returned channel. Implementations must honor ctx during channel
// sends so cancellation does not leak the producer goroutine.
type Agent interface {
Query(*Session, *Query) (chan Result, error)
Query(ctx context.Context, session *Session, query *Query) (chan Result, error)
Name() string
}

// SendResult delivers r on ch unless ctx is cancelled first. It returns
// false when the caller should stop producing, freeing producer goroutines
// blocked on send when no reader remains downstream.
func SendResult(ctx context.Context, ch chan<- Result, r Result) bool {
select {
case <-ctx.Done():
return false
case ch <- r:
return true
}
}
21 changes: 12 additions & 9 deletions sources/agent/binaryedge/binaryedge.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package binaryedge

import (
"context"
"encoding/json"
"errors"
"fmt"
Expand All @@ -21,24 +22,24 @@ func (agent *Agent) Name() string {
return "binaryedge"
}

func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan sources.Result, error) {
func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) {
if session.Keys.BinaryEdgeToken == "" {
return nil, errors.New("empty binaryedge token")
}

results := make(chan sources.Result)
go func() {
defer close(results)
agent.query(session, query.Query, results)
agent.query(ctx, session, query.Query, results)
}()

return results, nil
}

func (agent *Agent) query(session *sources.Session, searchQuery string, results chan sources.Result) {
resp, err := agent.queryURL(session, URL, searchQuery)
func (agent *Agent) query(ctx context.Context, session *sources.Session, searchQuery string, results chan sources.Result) {
resp, err := agent.queryURL(ctx, session, URL, searchQuery)
if err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
return
}
defer func() {
Expand All @@ -47,7 +48,7 @@ func (agent *Agent) query(session *sources.Session, searchQuery string, results

var apiResponse BinaryedgeResponse
if err := json.NewDecoder(resp.Body).Decode(&apiResponse); err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
return
}

Expand All @@ -61,14 +62,16 @@ func (agent *Agent) query(session *sources.Session, searchQuery string, results
if raw, err := json.Marshal(item); err == nil {
output.Raw = raw
}
results <- output
if !sources.SendResult(ctx, results, output) {
return
}
}
}
}

func (agent *Agent) queryURL(session *sources.Session, baseURL, searchQuery string) (*http.Response, error) {
func (agent *Agent) queryURL(ctx context.Context, session *sources.Session, baseURL, searchQuery string) (*http.Response, error) {
urlWithQuery := fmt.Sprintf(baseURL, url.QueryEscape(searchQuery))
request, err := sources.NewHTTPRequest(http.MethodGet, urlWithQuery, nil)
request, err := sources.NewHTTPRequest(ctx, http.MethodGet, urlWithQuery, nil)
if err != nil {
return nil, err
}
Expand Down
38 changes: 18 additions & 20 deletions sources/agent/censys/censys.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,11 @@ func (agent *Agent) Name() string {
return "censys"
}

func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan sources.Result, error) {
func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) {
if session.Keys.CensysToken == "" || session.Keys.CensysOrgId == "" {
return nil, errors.New("empty censys keys")
}

// Create the Censys SDK client once
s := censyssdkgo.New(
censyssdkgo.WithOrganizationID(session.Keys.CensysOrgId),
censyssdkgo.WithSecurity(session.Keys.CensysToken),
Expand All @@ -44,12 +43,15 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
var numberOfResults int
nextCursor := ""
for {
if ctx.Err() != nil {
return
}
censysRequest := &CensysRequest{
Query: query.Query,
PerPage: MaxPerPage,
Cursor: nextCursor,
}
censysResponse := agent.query(session, s, censysRequest, results)
censysResponse := agent.query(ctx, s, censysRequest, results)
if censysResponse == nil {
break
}
Expand All @@ -69,9 +71,7 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
return results, nil
}

func (agent *Agent) queryURL(s *censyssdkgo.SDK, censysRequest *CensysRequest) (*operations.V3GlobaldataSearchQueryResponse, error) {
ctx := context.Background()

func (agent *Agent) queryURL(ctx context.Context, s *censyssdkgo.SDK, censysRequest *CensysRequest) (*operations.V3GlobaldataSearchQueryResponse, error) {
return s.GlobalData.Search(ctx, operations.V3GlobaldataSearchQueryRequest{
SearchQueryInputBody: components.SearchQueryInputBody{
PageSize: censyssdkgo.Int64(int64(censysRequest.PerPage)),
Expand All @@ -81,37 +81,35 @@ func (agent *Agent) queryURL(s *censyssdkgo.SDK, censysRequest *CensysRequest) (
})
}

func (agent *Agent) query(session *sources.Session, s *censyssdkgo.SDK, censysRequest *CensysRequest, results chan sources.Result) *operations.V3GlobaldataSearchQueryResponse {
// query certificates
resp, err := agent.queryURL(s, censysRequest)
func (agent *Agent) query(ctx context.Context, s *censyssdkgo.SDK, censysRequest *CensysRequest, results chan sources.Result) *operations.V3GlobaldataSearchQueryResponse {
resp, err := agent.queryURL(ctx, s, censysRequest)
if err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
// httputil.DrainResponseBody(resp)
sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
return nil
}

if result := resp.ResponseEnvelopeSearchQueryResponse.Result; result != nil {
for _, censysResult := range result.Hits {

for _, host := range censysResult.WebpropertyV1.Resource.Endpoints {
result := sources.Result{Source: agent.Name()}
out := sources.Result{Source: agent.Name()}
if host.IP != nil {
result.IP = *host.IP
out.IP = *host.IP
}
if host.Hostname != nil {
result.Host = *host.Hostname
out.Host = *host.Hostname
}
if host.Port != nil {
result.Port = *host.Port
out.Port = *host.Port
}
if host.HTTP != nil && host.HTTP.URI != nil {
result.Url = *host.HTTP.URI
out.Url = *host.HTTP.URI
}
raw, _ := json.Marshal(host)
result.Raw = raw
results <- result
out.Raw = raw
if !sources.SendResult(ctx, results, out) {
return resp
}
}

}
}

Expand Down
25 changes: 15 additions & 10 deletions sources/agent/criminalip/criminalip.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package criminalip

import (
"context"
"encoding/json"
"fmt"
"net/http"
Expand All @@ -23,7 +24,7 @@ func (agent *Agent) Name() string {
return "criminalip"
}

func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan sources.Result, error) {
func (agent *Agent) Query(ctx context.Context, session *sources.Session, query *sources.Query) (chan sources.Result, error) {
if session.Keys.CriminalIPToken == "" {
return nil, errors.New("empty criminalip keys")
}
Expand All @@ -36,12 +37,15 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
currentPage := 0

for {
if ctx.Err() != nil {
return
}
criminalipRequest := &CriminalIPRequest{
Query: query.Query,
Offset: currentPage,
}

criminalipResponse := agent.query(URL, session, criminalipRequest, results)
criminalipResponse := agent.query(ctx, URL, session, criminalipRequest, results)
if criminalipResponse == nil {
break
}
Expand All @@ -65,28 +69,27 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
return results, nil
}

func (agent *Agent) queryURL(session *sources.Session, URL string, criminalipRequest *CriminalIPRequest) (*http.Response, error) {
func (agent *Agent) queryURL(ctx context.Context, session *sources.Session, URL string, criminalipRequest *CriminalIPRequest) (*http.Response, error) {
criminalipURL := fmt.Sprintf(URL, url.QueryEscape(criminalipRequest.Query), criminalipRequest.Offset)

request, err := sources.NewHTTPRequest(http.MethodGet, criminalipURL, nil)
request, err := sources.NewHTTPRequest(ctx, http.MethodGet, criminalipURL, nil)
if err != nil {
return nil, err
}
request.Header.Set("x-api-key", session.Keys.CriminalIPToken)
return session.Do(request, agent.Name())
}

func (agent *Agent) query(URL string, session *sources.Session, criminalipRequest *CriminalIPRequest, results chan sources.Result) *CriminalIPResponse {
// query certificates
resp, err := agent.queryURL(session, URL, criminalipRequest)
func (agent *Agent) query(ctx context.Context, URL string, session *sources.Session, criminalipRequest *CriminalIPRequest, results chan sources.Result) *CriminalIPResponse {
resp, err := agent.queryURL(ctx, session, URL, criminalipRequest)
if err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
return nil
}

criminalipResponse := &CriminalIPResponse{}
if err := json.NewDecoder(resp.Body).Decode(criminalipResponse); err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
sources.SendResult(ctx, results, sources.Result{Source: agent.Name(), Error: err})
return nil
}
if criminalipResponse.Status == http.StatusOK && criminalipResponse.Data.Count > 0 {
Expand All @@ -97,7 +100,9 @@ func (agent *Agent) query(URL string, session *sources.Session, criminalipReques
result.Host = criminalipResult.Domain
raw, _ := json.Marshal(criminalipResult)
result.Raw = raw
results <- result
if !sources.SendResult(ctx, results, result) {
return criminalipResponse
}
}
}

Expand Down
Loading
Loading