Merge remote-tracking branch 'origin' into dev

v2/examples/simple.go

@@ -5,12 +5,13 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"path"
+	"path/filepath"
 	"time"
 
 	"github.com/logrusorgru/aurora"
 
 	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
@@ -47,7 +48,7 @@ func main() {
 	protocolstate.Init(defaultOpts)
 	protocolinit.Init(defaultOpts)
 
-	defaultOpts.IncludeIds = goflags.StringSlice{"cname-service"}
+	defaultOpts.IncludeIds = goflags.StringSlice{"cname-service", "tech-detect"}
 	defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags
 
 	interactOpts := interactsh.DefaultOptions(outputWriter, reportingClient, mockProgress)
@@ -58,7 +59,7 @@ func main() {
 	defer interactClient.Close()
 
 	home, _ := os.UserHomeDir()
-	catalog := disk.NewCatalog(path.Join(home, "nuclei-templates"))
+	catalog := disk.NewCatalog(filepath.Join(home, "nuclei-templates"))
 	executerOpts := protocols.ExecutorOptions{
 		Output:          outputWriter,
 		Options:         defaultOpts,
@@ -86,9 +87,20 @@ func main() {
 	}
 	store.Load()
 
+	// flat input without probe
 	inputArgs := []*contextargs.MetaInput{{Input: "docs.hackerone.com"}}
-
 	input := &inputs.SimpleInputProvider{Inputs: inputArgs}
+
+	httpxOptions := httpx.DefaultOptions
+	httpxOptions.Timeout = 5 * time.Second
+	httpxClient, err := httpx.New(&httpxOptions)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// use httpx to probe the URL => https://scanme.sh
+	input.SetWithProbe("scanme.sh", httpxClient)
+
 	_ = engine.Execute(store.Templates(), input)
 	engine.WorkPool().Wait() // Wait for the scan to finish
 }

v2/internal/runner/inputs.go

@@ -1,18 +1,16 @@
 package runner
 
 import (
-	"fmt"
-	"net/http"
-	"strings"
 	"sync/atomic"
 	"time"
 
-	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
 	"github.com/projectdiscovery/httpx/common/httpx"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 	"github.com/remeh/sizedwaitgroup"
 )
 
@@ -25,7 +23,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create temporary input file")
 	}
-
 	gologger.Info().Msgf("Running httpx on input host")
 
 	var bulkSize = probeBulkSize
@@ -45,15 +42,15 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	swg := sizedwaitgroup.New(bulkSize)
 	count := int32(0)
 	r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
-		if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") {
+		if stringsutil.HasPrefixAny(value.Input, "http://", "https://") {
 			return true
 		}
 
 		swg.Add()
 		go func(input *contextargs.MetaInput) {
 			defer swg.Done()
 
-			if result := probeURL(input.Input, httpxClient); result != "" {
+			if result := utils.ProbeURL(input.Input, httpxClient); result != "" {
 				atomic.AddInt32(&count, 1)
 				_ = hm.Set(input.Input, []byte(result))
 			}
@@ -65,27 +62,3 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	gologger.Info().Msgf("Found %d URL from httpx", atomic.LoadInt32(&count))
 	return hm, nil
 }
-
-var (
-	httpSchemes = []string{"https", "http"}
-)
-
-// probeURL probes the scheme for a URL. first HTTPS is tried
-// and if any errors occur http is tried. If none succeeds, probing
-// is abandoned for such URLs.
-func probeURL(input string, httpxclient *httpx.HTTPX) string {
-	for _, scheme := range httpSchemes {
-		formedURL := fmt.Sprintf("%s://%s", scheme, input)
-		req, err := httpxclient.NewRequest(http.MethodHead, formedURL)
-		if err != nil {
-			continue
-		}
-		req.Header.Set("User-Agent", uarand.GetRandom())
-
-		if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil {
-			continue
-		}
-		return formedURL
-	}
-	return ""
-}

v2/pkg/core/inputs/inputs.go

@@ -1,6 +1,10 @@
 package inputs
 
-import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+import (
+	"github.com/projectdiscovery/httpx/common/httpx"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+)
 
 type SimpleInputProvider struct {
 	Inputs []*contextargs.MetaInput
@@ -24,3 +28,12 @@ func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) b
 func (s *SimpleInputProvider) Set(value string) {
 	s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: value})
 }
+
+// SetWithProbe adds item to input provider with http probing
+func (s *SimpleInputProvider) SetWithProbe(value string, httpxClient *httpx.HTTPX) {
+	valueToAppend := value
+	if result := utils.ProbeURL(value, httpxClient); result != "" {
+		valueToAppend = result
+	}
+	s.Inputs = append(s.Inputs, &contextargs.MetaInput{Input: valueToAppend})
+}

v2/pkg/utils/http_probe.go

@@ -0,0 +1,33 @@
+package utils
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/corpix/uarand"
+	"github.com/projectdiscovery/httpx/common/httpx"
+)
+
+var (
+	HttpSchemes = []string{"https", "http"}
+)
+
+// probeURL probes the scheme for a URL. first HTTPS is tried
+// and if any errors occur http is tried. If none succeeds, probing
+// is abandoned for such URLs.
+func ProbeURL(input string, httpxclient *httpx.HTTPX) string {
+	for _, scheme := range HttpSchemes {
+		formedURL := fmt.Sprintf("%s://%s", scheme, input)
+		req, err := httpxclient.NewRequest(http.MethodHead, formedURL)
+		if err != nil {
+			continue
+		}
+		req.Header.Set("User-Agent", uarand.GetRandom())
+
+		if _, err = httpxclient.Do(req, httpx.UnsafeOptions{}); err != nil {
+			continue
+		}
+		return formedURL
+	}
+	return ""
+}