add -dast flag and multiple bug fixes for dast templates (#4941)

* add default get method * remove residual payload logic from old implementation * fuzz: clone current state of component * fuzz: bug fix stacking of payloads in multiple mode * improve stdout template loading stats * stdout: force display warnings if no templates are loaded * update flags in README.md * quote non-ascii chars in extractor output * aws request signature can only be used in signed & verified tmpls * deprecate request signature * remove logic related to deprecated fuzzing input * update test to use ordered params * fix interactsh-url lazy eval: #4946 * output: skip unnecessary updates when unescaping * updates as per requested changes
projectdiscovery · Mar 29, 2024 · e88889b · e88889b
1 parent 78300e3
commit e88889b
Show file tree

Hide file tree

Showing 27 changed files with 216 additions and 274 deletions.
diff --git a/README.md b/README.md
@@ -221,7 +221,8 @@ INTERACTSH:
 FUZZING:
    -ft, -fuzzing-type string  overrides fuzzing type set in template (replace, prefix, postfix, infix)
    -fm, -fuzzing-mode string  overrides fuzzing mode set in template (multiple, single)
-   -fuzz                      enable loading fuzzing templates
+   -fuzz                      enable loading fuzzing templates (Deprecated: use -dast instead)
+   -dast                      only run DAST templates
 
 UNCOVER:
    -uc, -uncover                  enable uncover engine

diff --git a/cmd/integration-test/fuzz.go b/cmd/integration-test/fuzz.go
@@ -21,8 +21,6 @@ var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
-	{Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}},
-	{Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}},
 	// for fuzzing we should prioritize adding test case related backend
 	// logic in fuzz playground server instead of adding them here
 	{Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}},
@@ -176,52 +174,3 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 2)
 }
-
-type FuzzHeaderBasic struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *FuzzHeaderBasic) Execute(filePath string) error {
-	router := httprouter.New()
-	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		host := r.Header.Get("Origin")
-		// redirect to different domain
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprintf(w, "<html><body><a href="+host+">Click Here</a></body></html>")
-	})
-	ts := httptest.NewTLSServer(router)
-	defer ts.Close()
-
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(got, 1)
-}
-
-type FuzzHeaderMultiple struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *FuzzHeaderMultiple) Execute(filePath string) error {
-	router := httprouter.New()
-	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		host1 := r.Header.Get("Origin")
-		host2 := r.Header.Get("X-Forwared-For")
-
-		fmt.Printf("host1: %s, host2: %s\n", host1, host2)
-		if host1 == host2 && host2 == "secret.local" {
-			w.WriteHeader(http.StatusOK)
-			fmt.Fprintf(w, "welcome! to secret admin panel")
-			return
-		}
-		// redirect to different domain
-		w.WriteHeader(http.StatusForbidden)
-	})
-	ts := httptest.NewTLSServer(router)
-	defer ts.Close()
-
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(got, 1)
-}
diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go
@@ -186,6 +186,7 @@ func readConfig() *goflags.FlagSet {
 	// when true updates nuclei binary to latest version
 	var updateNucleiBinary bool
 	var pdcpauth string
+	var fuzzFlag bool
 
 	flagSet := goflags.NewFlagSet()
 	flagSet.CaseSensitive = true
@@ -313,7 +314,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("fuzzing", "Fuzzing",
 		flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"),
 		flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
-		flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable loading fuzzing templates"),
+		flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
+		flagSet.BoolVar(&options.DAST, "dast", false, "only run DAST templates"),
 	)
 
 	flagSet.CreateGroup("uncover", "Uncover",
@@ -436,6 +438,12 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 	goflags.DisableAutoConfigMigration = true
 	_ = flagSet.Parse()
 
+	// when fuzz flag is enabled, set the dast flag to true
+	if fuzzFlag {
+		// backwards compatibility for fuzz flag
+		options.DAST = true
+	}
+
 	// api key hierarchy: cli flag > env var > .pdcp/credential file
 	if pdcpauth == "true" {
 		runner.AuthWithPDCP()

diff --git a/integration_tests/fuzz/fuzz-header-basic.yaml b/integration_tests/fuzz/fuzz-header-basic.yaml
diff --git a/integration_tests/fuzz/fuzz-header-multiple.yaml b/integration_tests/fuzz/fuzz-header-multiple.yaml
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
@@ -475,10 +475,9 @@ func (r *Runner) RunEnumeration() error {
 
 	// If using input-file flags, only load http fuzzing based templates.
 	loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts)
-	if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.FuzzTemplates {
+	if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST {
 		// if input type is not list (implicitly enable fuzzing)
-		r.options.FuzzTemplates = true
-		loaderConfig.OnlyLoadHTTPFuzzing = true
+		r.options.DAST = true
 	}
 	store, err := loader.New(loaderConfig)
 	if err != nil {
@@ -640,19 +639,27 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	stats.Display(templates.SyntaxWarningStats)
 	stats.Display(templates.SyntaxErrorStats)
 	stats.Display(templates.RuntimeWarningsStats)
-	if r.options.Verbose {
+	tmplCount := len(store.Templates())
+	workflowCount := len(store.Workflows())
+	if r.options.Verbose || (tmplCount == 0 && workflowCount == 0) {
 		// only print these stats in verbose mode
-		stats.DisplayAsWarning(templates.HeadlessFlagWarningStats)
-		stats.DisplayAsWarning(templates.CodeFlagWarningStats)
-		stats.DisplayAsWarning(templates.TemplatesExecutedStats)
-		stats.DisplayAsWarning(templates.HeadlessFlagWarningStats)
-		stats.DisplayAsWarning(templates.CodeFlagWarningStats)
-		stats.DisplayAsWarning(templates.FuzzFlagWarningStats)
-		stats.DisplayAsWarning(templates.TemplatesExecutedStats)
+		stats.ForceDisplayWarning(templates.ExcludedHeadlessTmplStats)
+		stats.ForceDisplayWarning(templates.ExcludedCodeTmplStats)
+		stats.ForceDisplayWarning(templates.ExludedDastTmplStats)
+		stats.ForceDisplayWarning(templates.TemplatesExcludedStats)
 	}
 
-	stats.DisplayAsWarning(templates.UnsignedCodeWarning)
+	if tmplCount == 0 && workflowCount == 0 {
+		// if dast flag is used print explicit warning
+		if r.options.DAST {
+			gologger.DefaultLogger.Print().Msgf("[%v] No DAST templates found", aurora.BrightYellow("WRN"))
+		}
+		stats.ForceDisplayWarning(templates.SkippedCodeTmplTamperedStats)
+	} else {
+		stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats)
+	}
 	stats.ForceDisplayWarning(templates.SkippedUnsignedStats)
+	stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats)
 
 	cfg := config.DefaultConfig
 
@@ -666,25 +673,27 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 		}
 	}
 
-	if len(store.Templates()) > 0 {
-		gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions()))
-		gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates()))
-	}
-	if len(store.Workflows()) > 0 {
-		gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows()))
-	}
-	for k, v := range templates.SignatureStats {
-		value := v.Load()
-		if k == templates.Unsigned && value > 0 {
-			// adjust skipped unsigned templates via code or -dut flag
-			value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats))
-			value = value - uint64(stats.GetValue(templates.CodeFlagWarningStats))
+	if tmplCount > 0 || workflowCount > 0 {
+		if len(store.Templates()) > 0 {
+			gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions()))
+			gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates()))
+		}
+		if len(store.Workflows()) > 0 {
+			gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows()))
 		}
-		if value > 0 {
-			if k != templates.Unsigned {
-				gologger.Info().Msgf("Executing %d signed templates from %s", value, k)
-			} else if !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning {
-				gologger.Print().Msgf("[%v] Loaded %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value)
+		for k, v := range templates.SignatureStats {
+			value := v.Load()
+			if k == templates.Unsigned && value > 0 {
+				// adjust skipped unsigned templates via code or -dut flag
+				value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats))
+				value = value - uint64(stats.GetValue(templates.ExcludedCodeTmplStats))
+			}
+			if value > 0 {
+				if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning {
+					gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value)
+				} else {
+					gologger.Info().Msgf("Executing %d signed templates from %s", value, k)
+				}
 			}
 		}
 	}

diff --git a/lib/config.go b/lib/config.go
@@ -376,10 +376,18 @@ func LoadSecretsFromFile(files []string, prefetch bool) NucleiSDKOptions {
 	}
 }
 
-// EnableFuzzTemplates allows enabling template fuzzing
-func EnableFuzzTemplates() NucleiSDKOptions {
+// DASTMode only run DAST templates
+func DASTMode() NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
-		e.opts.FuzzTemplates = true
+		e.opts.DAST = true
+		return nil
+	}
+}
+
+// SignedTemplatesOnly only run signed templates and disabled loading all unsigned templates
+func SignedTemplatesOnly() NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.opts.DisableUnsignedTemplates = true
 		return nil
 	}
 }
diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go
@@ -61,8 +61,6 @@ type Config struct {
 
 	Catalog         catalog.Catalog
 	ExecutorOptions protocols.ExecutorOptions
-
-	OnlyLoadHTTPFuzzing bool
 }
 
 // Store is a storage for loaded nuclei templates
@@ -405,41 +403,50 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 					stats.Increment(templates.SkippedUnsignedStats)
 					continue
 				}
-				if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
+				// if template has request signature like aws then only signed and verified templates are allowed
+				if parsed.UsesRequestSignature() && !parsed.Verified {
+					stats.Increment(templates.SkippedRequestSignatureStats)
+					continue
+				}
+				// DAST only templates
+				if store.config.ExecutorOptions.Options.DAST {
+					// check if the template is a DAST template
+					if parsed.IsFuzzing() {
+						loadedTemplates = append(loadedTemplates, parsed)
+					}
+				} else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
 					// donot include headless template in final list if headless flag is not set
-					stats.Increment(templates.HeadlessFlagWarningStats)
+					stats.Increment(templates.ExcludedHeadlessTmplStats)
 					if config.DefaultConfig.LogAllEvents {
 						gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
 					}
 				} else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
 					// donot include 'Code' protocol custom template in final list if code flag is not set
-					stats.Increment(templates.CodeFlagWarningStats)
+					stats.Increment(templates.ExcludedCodeTmplStats)
 					if config.DefaultConfig.LogAllEvents {
 						gologger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
 					}
 				} else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 {
 					// donot include unverified 'Code' protocol custom template in final list
-					stats.Increment(templates.UnsignedCodeWarning)
+					stats.Increment(templates.SkippedCodeTmplTamperedStats)
 					// these will be skipped so increment skip counter
 					stats.Increment(templates.SkippedUnsignedStats)
 					if config.DefaultConfig.LogAllEvents {
 						gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath)
 					}
-				} else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.FuzzTemplates {
-					stats.Increment(templates.FuzzFlagWarningStats)
+				} else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
+					stats.Increment(templates.ExludedDastTmplStats)
 					if config.DefaultConfig.LogAllEvents {
-						gologger.Print().Msgf("[%v] Fuzz flag is required for fuzzing template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
+						gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
 					}
-				} else if store.config.OnlyLoadHTTPFuzzing && !parsed.IsFuzzing() {
-					gologger.Warning().Msgf("Non-Fuzzing template '%s' can only be run on list input mode targets\n", templatePath)
 				} else {
 					loadedTemplates = append(loadedTemplates, parsed)
 				}
 			}
 		}
 		if err != nil {
 			if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
-				stats.Increment(templates.TemplatesExecutedStats)
+				stats.Increment(templates.TemplatesExcludedStats)
 				if cfg.DefaultConfig.LogAllEvents {
 					gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
 				}

diff --git a/pkg/fuzz/component/body.go b/pkg/fuzz/component/body.go
@@ -141,3 +141,10 @@ func (b *Body) Rebuild() (*retryablehttp.Request, error) {
 	cloned.Header.Set("Content-Length", strconv.Itoa(len(encoded)))
 	return cloned, nil
 }
+
+func (b *Body) Clone() Component {
+	return &Body{
+		value: b.value.Clone(),
+		req:   b.req.Clone(context.Background()),
+	}
+}