From bb98eced070f4ae137b8cd2a7f887611bc1b9c93 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 3 Jul 2023 18:42:25 +0530 Subject: [PATCH 1/2] dep update --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index b32dfa33dc..c526a72ac7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -66,7 +66,7 @@ require ( github.com/klauspost/compress v1.16.6 github.com/labstack/echo/v4 v4.10.2 github.com/mholt/archiver v3.1.1+incompatible - github.com/projectdiscovery/dsl v0.0.12 + github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/gologger v1.1.10 diff --git a/v2/go.sum b/v2/go.sum index 0cb3adfa12..9045a53318 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -395,8 +395,8 @@ github.com/projectdiscovery/cdncheck v1.0.6 h1:bjo4oxCD1Y5972ow0LWCjUpO8KOO12j6u github.com/projectdiscovery/cdncheck v1.0.6/go.mod h1:NN0QRfxBzUVZJoS0lN37spElCOXHzFuvq1yg5RhTxCE= github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9ZjAnDatybhLw= github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= -github.com/projectdiscovery/dsl v0.0.12 h1:F3S94FKyakMMtRNuob+HbW0XmibBE3zwWBw+b10x2gg= -github.com/projectdiscovery/dsl v0.0.12/go.mod h1:UQxYzKD9oy/xs86rHMfCcVb+JoPJ8qUhxm9AejdsvFw= +github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026 h1:wsV4dRVDJZlWu+aEGxs3Husnh3x7wCAZOifDpOEbXmY= +github.com/projectdiscovery/dsl v0.0.13-0.20230703115450-39d83b5a7026/go.mod h1:Zwn4CPdOxpQogu1rU2Z4kxbxUJdbLCQ+34ZK7AZH0i0= github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs= github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= From ac51b571aeb7fb1cda319ed30323be2946db83cf Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 22 Jul 2023 00:49:52 +0200 Subject: [PATCH 2/2] adding set with httpx probe (#3955) --- v2/examples/simple.go | 20 ++++++++++++++++---- v2/internal/runner/inputs.go | 35 ++++------------------------------- v2/pkg/core/inputs/inputs.go | 15 ++++++++++++++- v2/pkg/utils/http_probe.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 v2/pkg/utils/http_probe.go diff --git a/v2/examples/simple.go b/v2/examples/simple.go index ea7fe130c1..e1f9a4cd4d 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -5,12 +5,13 @@ import ( "fmt" "log" "os" - "path" + "path/filepath" "time" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" @@ -47,7 +48,7 @@ func main() { protocolstate.Init(defaultOpts) protocolinit.Init(defaultOpts) - defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"} + defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"} defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress) @@ -58,7 +59,7 @@ func main() { defer interactClient.Close() home, _ := os.UserHomeDir() - catalog := disk.NewCatalog(path.Join(home, "nuclei-templates")) + catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates")) executerOpts := protocols.ExecutorOptions{ Output: outputWriter, Options: defaultOpts, @@ -86,9 +87,20 @@ func main() { } store.Load() + // flat input without probe inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}} - input := &inputs.SimpleInputProvider{Inputs: inputArgs} + + httpxOptions := httpx.DefaultOptions + httpxOptions.Timeout = 5 * time.Second + httpxClient, err := httpx.New(&httpxOptions) + if err != nil { + log.Fatal(err) + } + + // use httpx to probe the URL => https://scanme.sh + input.SetWithProbe("scanme.sh", httpxClient) + _ = engine.Execute(store.Templates(), input) engine.WorkPool().Wait() // Wait for the scan to finish } diff --git a/v2/internal/runner/inputs.go b/v2/internal/runner/inputs.go index 1bce3f8797..191972a033 100644 --- a/v2/internal/runner/inputs.go +++ b/v2/internal/runner/inputs.go @@ -1,18 +1,16 @@ package runner import ( - "fmt" - "net/http" - "strings" "sync/atomic" "time" - "github.com/corpix/uarand" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" + stringsutil "github.com/projectdiscovery/utils/strings" "github.com/remeh/sizedwaitgroup" ) @@ -25,7 +23,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { if err != nil { return nil, errors.Wrap(err, "could not create temporary input file") } - gologger.Info().Msgf("Running httpx on input host") var bulkSize = probeBulkSize @@ -45,7 +42,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { swg := sizedwaitgroup.New(bulkSize) count := int32(0) r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { - if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") { + if stringsutil.HasPrefixAny(value.Input, "http://", "https://") { return true } @@ -53,7 +50,7 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { go func(input *contextargs.MetaInput) { defer swg.Done() - if result := probeURL(input.Input, httpxClient); result != "" { + if result := utils.ProbeURL(input.Input, httpxClient); result != "" { atomic.AddInt32(&count, 1) _ = hm.Set(input.Input, []byte(result)) } @@ -65,27 +62,3 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count)) return hm, nil } - -var ( - httpSchemes = []string{"https", "http"} -) - -// probeURL probes the scheme for a URL. first HTTPS is tried -// and if any errors occur http is tried. If none succeeds, probing -// is abandoned for such URLs. -func probeURL(input string, httpxclient *httpx.HTTPX) string { - for _, scheme := range httpSchemes { - formedURL := fmt.Sprintf("%s://%s", scheme, input) - req, err := httpxclient.NewRequest(http.MethodHead, formedURL) - if err != nil { - continue - } - req.Header.Set("User-Agent", uarand.GetRandom()) - - if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { - continue - } - return formedURL - } - return "" -} diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go index bb75b6e9cf..5e04d71284 100644 --- a/v2/pkg/core/inputs/inputs.go +++ b/v2/pkg/core/inputs/inputs.go @@ -1,6 +1,10 @@ package inputs -import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +import ( + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" +) type SimpleInputProvider struct { Inputs []*contextargs.MetaInput @@ -24,3 +28,12 @@ func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) b func (s *SimpleInputProvider) Set(value string) { s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value}) } + +// SetWithProbe adds item to input provider with http probing +func (s *SimpleInputProvider) SetWithProbe(value string, httpxClient *httpx.HTTPX) { + valueToAppend := value + if result := utils.ProbeURL(value, httpxClient); result != "" { + valueToAppend = result + } + s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: valueToAppend}) +} diff --git a/v2/pkg/utils/http_probe.go b/v2/pkg/utils/http_probe.go new file mode 100644 index 0000000000..f79567d759 --- /dev/null +++ b/v2/pkg/utils/http_probe.go @@ -0,0 +1,33 @@ +package utils + +import ( + "fmt" + "net/http" + + "github.com/corpix/uarand" + "github.com/projectdiscovery/httpx/common/httpx" +) + +var ( + HttpSchemes = []string{"https", "http"} +) + +// probeURL probes the scheme for a URL. first HTTPS is tried +// and if any errors occur http is tried. If none succeeds, probing +// is abandoned for such URLs. +func ProbeURL(input string, httpxclient *httpx.HTTPX) string { + for _, scheme := range HttpSchemes { + formedURL := fmt.Sprintf("%s://%s", scheme, input) + req, err := httpxclient.NewRequest(http.MethodHead, formedURL) + if err != nil { + continue + } + req.Header.Set("User-Agent", uarand.GetRandom()) + + if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil { + continue + } + return formedURL + } + return "" +}