Fuzzing additions & enhancements

cmd/nuclei/main.go

@@ -323,6 +323,9 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
 		flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
 		flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
+		flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"),
+		flagSet.IntVar(&options.FuzzParamFrequency, "fuzz-param-frequency", 10, "frequency of uninteresting parameters for fuzzing before skipping"),
+		flagSet.StringVarP(&options.FuzzAggressionLevel, "fuzz-aggression", "fa", "low", "fuzzing aggression level controls payload count for fuzz (low, medium, high)"),
 	)
 
 	flagSet.CreateGroup("uncover", "Uncover",

integration_tests/fuzz/fuzz-path-sqli.yaml

@@ -15,21 +15,18 @@ http:
       - type: dsl
         dsl:
           - 'method == "GET"'
-          - regex("/(.*?/)([0-9]+)(/.*)?",path)
         condition: and
 
     payloads:
       pathsqli:
-        - "'OR1=1"
         - '%20OR%20True'
 
     fuzzing:
       - part: path
-        type: replace-regex
+        type: postfix
         mode: single
-        replace-regex: '/(.*?/)([0-9]+)(/.*)?'
         fuzz:
-          - '/${1}${2}{{pathsqli}}${3}'
+          - '{{pathsqli}}'
 
     matchers:
       - type: status

internal/runner/runner.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v3/internal/pdcp"
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
@@ -74,21 +75,22 @@ var (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
-	output           output.Writer
-	interactsh       *interactsh.Client
-	options          *types.Options
-	projectFile      *projectfile.ProjectFile
-	catalog          catalog.Catalog
-	progress         progress.Progress
-	colorizer        aurora.Aurora
-	issuesClient     reporting.Client
-	browser          *engine.Browser
-	rateLimiter      *ratelimit.Limiter
-	hostErrors       hosterrorscache.CacheInterface
-	resumeCfg        *types.ResumeCfg
-	pprofServer      *http.Server
-	pdcpUploadErrMsg string
-	inputProvider    provider.InputProvider
+	output             output.Writer
+	interactsh         *interactsh.Client
+	options            *types.Options
+	projectFile        *projectfile.ProjectFile
+	catalog            catalog.Catalog
+	progress           progress.Progress
+	colorizer          aurora.Aurora
+	issuesClient       reporting.Client
+	browser            *engine.Browser
+	rateLimiter        *ratelimit.Limiter
+	hostErrors         hosterrorscache.CacheInterface
+	resumeCfg          *types.ResumeCfg
+	pprofServer        *http.Server
+	pdcpUploadErrMsg   string
+	inputProvider      provider.InputProvider
+	fuzzFrequencyCache *frequency.Tracker
 	//general purpose temporary directory
 	tmpDir          string
 	parser          parser.Parser
@@ -440,24 +442,28 @@ func (r *Runner) RunEnumeration() error {
 		r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 	}
 
+	fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency)
+	r.fuzzFrequencyCache = fuzzFreqCache
+
 	// Create the executor options which will be used throughout the execution
 	// stage by the nuclei engine modules.
 	executorOpts := protocols.ExecutorOptions{
-		Output:             r.output,
-		Options:            r.options,
-		Progress:           r.progress,
-		Catalog:            r.catalog,
-		IssuesClient:       r.issuesClient,
-		RateLimiter:        r.rateLimiter,
-		Interactsh:         r.interactsh,
-		ProjectFile:        r.projectFile,
-		Browser:            r.browser,
-		Colorizer:          r.colorizer,
-		ResumeCfg:          r.resumeCfg,
-		ExcludeMatchers:    excludematchers.New(r.options.ExcludeMatchers),
-		InputHelper:        input.NewHelper(),
-		TemporaryDirectory: r.tmpDir,
-		Parser:             r.parser,
+		Output:              r.output,
+		Options:             r.options,
+		Progress:            r.progress,
+		Catalog:             r.catalog,
+		IssuesClient:        r.issuesClient,
+		RateLimiter:         r.rateLimiter,
+		Interactsh:          r.interactsh,
+		ProjectFile:         r.projectFile,
+		Browser:             r.browser,
+		Colorizer:           r.colorizer,
+		ResumeCfg:           r.resumeCfg,
+		ExcludeMatchers:     excludematchers.New(r.options.ExcludeMatchers),
+		InputHelper:         input.NewHelper(),
+		TemporaryDirectory:  r.tmpDir,
+		Parser:              r.parser,
+		FuzzParamsFrequency: fuzzFreqCache,
 	}
 
 	if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" {
@@ -611,6 +617,7 @@ func (r *Runner) RunEnumeration() error {
 	if executorOpts.InputHelper != nil {
 		_ = executorOpts.InputHelper.Close()
 	}
+	r.fuzzFrequencyCache.Close()
 
 	// todo: error propagation without canonical straight error check is required by cloud?
 	// use safe dereferencing to avoid potential panics in case of previous unchecked errors

pkg/fuzz/component/path.go

@@ -2,10 +2,12 @@ package component
 
 import (
 	"context"
+	"strconv"
+	"strings"
 
-	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat"
 	"github.com/projectdiscovery/retryablehttp-go"
+	urlutil "github.com/projectdiscovery/utils/url"
 )
 
 // Path is a component for a request Path
@@ -31,13 +33,18 @@ func (q *Path) Name() string {
 // parsed component
 func (q *Path) Parse(req *retryablehttp.Request) (bool, error) {
 	q.req = req
-	q.value = NewValue(req.URL.Path)
+	q.value = NewValue("")
 
-	parsed, err := dataformat.Get(dataformat.RawDataFormat).Decode(q.value.String())
-	if err != nil {
-		return false, err
+	splitted := strings.Split(req.URL.Path, "/")
+	values := make(map[string]interface{})
+	for i := range splitted {
+		pathTillNow := strings.Join(splitted[:i+1], "/")
+		if pathTillNow == "" {
+			continue
+		}
+		values[strconv.Itoa(i)] = pathTillNow
 	}
-	q.value.SetParsed(parsed, dataformat.RawDataFormat)
+	q.value.SetParsed(dataformat.KVMap(values), "")
 	return true, nil
 }
 
@@ -56,7 +63,8 @@ func (q *Path) Iterate(callback func(key string, value interface{}) error) (err
 // SetValue sets a value in the component
 // for a key
 func (q *Path) SetValue(key string, value string) error {
-	if !q.value.SetParsedValue(key, value) {
+	escaped := urlutil.ParamEncode(value)
+	if !q.value.SetParsedValue(key, escaped) {
 		return ErrSetValue
 	}
 	return nil
@@ -73,13 +81,31 @@ func (q *Path) Delete(key string) error {
 // Rebuild returns a new request with the
 // component rebuilt
 func (q *Path) Rebuild() (*retryablehttp.Request, error) {
-	encoded, err := q.value.Encode()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not encode query")
+	originalValues := make(map[string]interface{})
+	splitted := strings.Split(q.req.URL.Path, "/")
+	for i := range splitted {
+		pathTillNow := strings.Join(splitted[:i+1], "/")
+		if pathTillNow == "" {
+			continue
+		}
+		originalValues[strconv.Itoa(i)] = pathTillNow
+	}
+
+	originalPath := q.req.URL.Path
+	lengthSplitted := len(q.value.parsed.Map)
+	for i := lengthSplitted; i > 0; i-- {
+		key := strconv.Itoa(i)
+		original := originalValues[key].(string)
+		new := q.value.parsed.Map[key].(string)
+		originalPath = strings.Replace(originalPath, original, new, 1)
 	}
+
+	rebuiltPath := originalPath
+
+	// Clone the request and update the path
 	cloned := q.req.Clone(context.Background())
-	if err := cloned.UpdateRelPath(encoded, true); err != nil {
-		cloned.URL.RawPath = encoded
+	if err := cloned.UpdateRelPath(rebuiltPath, true); err != nil {
+		cloned.URL.RawPath = rebuiltPath
 	}
 	return cloned, nil
 }

pkg/fuzz/component/path_test.go

@@ -28,10 +28,10 @@ func TestURLComponent(t *testing.T) {
 		return nil
 	})
 
-	require.Equal(t, []string{"value"}, keys, "unexpected keys")
+	require.Equal(t, []string{"1"}, keys, "unexpected keys")
 	require.Equal(t, []string{"/testpath"}, values, "unexpected values")
 
-	err = urlComponent.SetValue("value", "/newpath")
+	err = urlComponent.SetValue("1", "/newpath")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -40,7 +40,41 @@ func TestURLComponent(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	require.Equal(t, "/newpath", rebuilt.URL.Path, "unexpected URL path")
 	require.Equal(t, "https://example.com/newpath", rebuilt.URL.String(), "unexpected full URL")
 }
+
+func TestURLComponent_NestedPaths(t *testing.T) {
+	path := NewPath()
+	req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com/user/753/profile", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	found, err := path.Parse(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !found {
+		t.Fatal("expected path to be found")
+	}
+
+	isSet := false
+
+	_ = path.Iterate(func(key string, value interface{}) error {
+		if !isSet && value.(string) == "/user/753" {
+			isSet = true
+			if setErr := path.SetValue(key, "/user/753'"); setErr != nil {
+				t.Fatal(setErr)
+			}
+		}
+		return nil
+	})
+
+	newReq, err := path.Rebuild()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if newReq.URL.Path != "/user/753'/profile" {
+		t.Fatal("expected path to be modified")
+	}
+}

pkg/fuzz/component/query.go

@@ -61,6 +61,7 @@ func (q *Query) Iterate(callback func(key string, value interface{}) error) (err
 // SetValue sets a value in the component
 // for a key
 func (q *Query) SetValue(key string, value string) error {
+	// Is this safe?
 	if !q.value.SetParsedValue(key, value) {
 		return ErrSetValue
 	}

pkg/fuzz/execute.go

@@ -1,6 +1,7 @@
 package fuzz
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"regexp"
@@ -15,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/retryablehttp-go"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -45,6 +47,8 @@ type ExecuteRuleInput struct {
 	Values map[string]interface{}
 	// BaseRequest is the base http request for fuzzing rule
 	BaseRequest *retryablehttp.Request
+	// DisplayFuzzPoints is a flag to display fuzz points
+	DisplayFuzzPoints bool
 }
 
 // GeneratedRequest is a single generated request for rule
@@ -76,8 +80,9 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
 
 	var finalComponentList []component.Component
 	// match rule part with component name
+	displayDebugFuzzPoints := make(map[string]map[string]string)
 	for _, componentName := range component.Components {
-		if rule.partType != requestPartType && rule.Part != componentName {
+		if !(rule.Part == componentName || sliceutil.Contains(rule.Parts, componentName) || rule.partType == requestPartType) {
 			continue
 		}
 		component := component.New(componentName)
@@ -89,12 +94,25 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
 		if !discovered {
 			continue
 		}
+
 		// check rule applicable on this component
 		if !rule.checkRuleApplicableOnComponent(component) {
 			continue
 		}
+		// Debugging display for fuzz points
+		if input.DisplayFuzzPoints {
+			displayDebugFuzzPoints[componentName] = make(map[string]string)
+			_ = component.Iterate(func(key string, value interface{}) error {
+				displayDebugFuzzPoints[componentName][key] = fmt.Sprintf("%v", value)
+				return nil
+			})
+		}
 		finalComponentList = append(finalComponentList, component)
 	}
+	if len(displayDebugFuzzPoints) > 0 {
+		marshalled, _ := json.MarshalIndent(displayDebugFuzzPoints, "", "  ")
+		gologger.Info().Msgf("[%s] Fuzz points for %s [%s]\n%s\n", rule.options.TemplateID, input.Input.MetaInput.Input, input.BaseRequest.Method, string(marshalled))
+	}
 
 	if len(finalComponentList) == 0 {
 		return ErrRuleNotApplicable.Msgf("no component matched on this rule")
@@ -225,7 +243,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
 			if err != nil {
 				return err
 			}
-			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil {
+			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", ""); gotErr != nil {
 				return gotErr
 			}
 		}
@@ -261,8 +279,9 @@ func (rule *Rule) Compile(generator *generators.PayloadGenerator, options *proto
 		} else {
 			rule.partType = valueType
 		}
-	} else {
-		rule.partType = queryPartType
+	}
+	if rule.Part == "" && len(rule.Parts) == 0 {
+		return errors.Errorf("no part specified for rule")
 	}
 
 	if rule.Type != "" {