From 75dd6553ba94a26247feba9ea614f55b13c2591e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 1 May 2024 02:06:31 +0530 Subject: [PATCH 01/12] feat: added fuzzing output enhancements --- pkg/fuzz/execute.go | 4 +++- pkg/fuzz/parts.go | 9 +++++---- pkg/output/format_screen.go | 14 ++++++++++++++ pkg/output/output.go | 8 ++++++++ pkg/protocols/http/request_fuzz.go | 7 +++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index 23b3e6e976..bd1414638e 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -57,6 +57,8 @@ type GeneratedRequest struct { DynamicValues map[string]interface{} // Component is the component for the request Component component.Component + // Parameter being fuzzed + Parameter string } // Execute executes a fuzzing rule accepting a callback on which @@ -223,7 +225,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo if err != nil { return err } - if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); gotErr != nil { + if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil { return gotErr } } diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index 2796a7dc96..4c01135f66 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -68,7 +68,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { return qerr } // fmt.Printf("executed with value: %s\n", evaluated) @@ -90,7 +90,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS if err != nil { return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil { err = qerr return err } @@ -125,7 +125,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { return err } @@ -144,12 +144,13 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu } // execWithInput executes a rule with input via callback -func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component) error { +func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error { request := GeneratedRequest{ Request: httpReq, InteractURLs: interactURLs, DynamicValues: input.Values, Component: component, + Parameter: parameter, } if !input.Callback(request) { return types.ErrNoMoreRequests diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 2d6310df3f..0682447894 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -106,5 +106,19 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { } builder.WriteString("]") } + + // If it is a fuzzing output, enrich with additional + // metadata for the match. + if output.IsFuzzingResult { + builder.WriteString(" [") + builder.WriteString(output.FuzzingPosition) + builder.WriteRune(':') + builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String()) + builder.WriteString("]") + + builder.WriteString(" [") + builder.WriteString(output.FuzzingMethod) + builder.WriteString("]") + } return builder.Bytes() } diff --git a/pkg/output/output.go b/pkg/output/output.go index 4a85c32480..044d164f1f 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -175,6 +175,14 @@ type ResultEvent struct { // must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true ReqURLPattern string `json:"req_url_pattern,omitempty"` + // Fields related to HTTP Fuzzing functionality of nuclei. + // The output contains additional fields when the result is + // for a fuzzing template. + IsFuzzingResult bool `json:"is_fuzzing_result,omitempty"` + FuzzingMethod string `json:"fuzzing_method,omitempty"` + FuzzingParameter string `json:"fuzzing_parameter,omitempty"` + FuzzingPosition string `json:"fuzzing_position,omitempty"` + FileToIndexPosition map[string]int `json:"-"` Error string `json:"error,omitempty"` } diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index 42065576ad..bc5f9caa9b 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -170,6 +170,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, } var gotMatches bool requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) { + for _, result := range event.Results { + result.IsFuzzingResult = true + result.FuzzingMethod = gr.Request.Method + result.FuzzingParameter = gr.Parameter + result.FuzzingPosition = gr.Component.Name() + } + if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, From 4a8f66f3ea81ee6843f7edcd95c2a6d9402fd83d Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 3 May 2024 16:56:12 +0530 Subject: [PATCH 02/12] changes as requested --- pkg/output/format_screen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 0682447894..433d631a35 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -109,7 +109,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { // If it is a fuzzing output, enrich with additional // metadata for the match. - if output.IsFuzzingResult { + if output.IsFuzzingResult && output.FuzzingParameter != "" { builder.WriteString(" [") builder.WriteString(output.FuzzingPosition) builder.WriteRune(':') From 03a3113c891b6494a3ec7ddf666e2decb1e49085 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 3 May 2024 18:10:41 +0530 Subject: [PATCH 03/12] misc --- pkg/output/format_screen.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index 433d631a35..9d0f03efaf 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -109,12 +109,14 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { // If it is a fuzzing output, enrich with additional // metadata for the match. - if output.IsFuzzingResult && output.FuzzingParameter != "" { - builder.WriteString(" [") - builder.WriteString(output.FuzzingPosition) - builder.WriteRune(':') - builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String()) - builder.WriteString("]") + if output.IsFuzzingResult { + if output.FuzzingParameter != "" { + builder.WriteString(" [") + builder.WriteString(output.FuzzingPosition) + builder.WriteRune(':') + builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String()) + builder.WriteString("]") + } builder.WriteString(" [") builder.WriteString(output.FuzzingMethod) From 927525beb4c2880348c2851b10b9da32e83969be Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 3 May 2024 23:37:44 +0530 Subject: [PATCH 04/12] feat: added dfp flag to display fuzz points + misc additions --- cmd/nuclei/main.go | 1 + pkg/fuzz/execute.go | 20 +++++++++++++++++++- pkg/fuzz/parts.go | 2 +- pkg/protocols/http/request_fuzz.go | 4 +++- pkg/testutils/fuzzplayground/server.go | 22 +++++++++++++++------- pkg/types/types.go | 2 ++ 6 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 242a902f55..4227202c84 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -319,6 +319,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"), flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), + flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), ) flagSet.CreateGroup("uncover", "Uncover", diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index bd1414638e..f05b634225 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -1,6 +1,7 @@ package fuzz import ( + "encoding/json" "fmt" "io" "regexp" @@ -45,6 +46,8 @@ type ExecuteRuleInput struct { Values map[string]interface{} // BaseRequest is the base http request for fuzzing rule BaseRequest *retryablehttp.Request + // DisplayFuzzPoints is a flag to display fuzz points + DisplayFuzzPoints bool } // GeneratedRequest is a single generated request for rule @@ -76,8 +79,9 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { var finalComponentList []component.Component // match rule part with component name + displayDebugFuzzPoints := make(map[string]map[string]string) for _, componentName := range component.Components { - if rule.partType != requestPartType && rule.Part != componentName { + if !(rule.Part == componentName || rule.partType == requestPartType) { continue } component := component.New(componentName) @@ -89,12 +93,26 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { if !discovered { continue } + // check rule applicable on this component if !rule.checkRuleApplicableOnComponent(component) { continue } + // Debugging display for fuzz points + if input.DisplayFuzzPoints { + displayDebugFuzzPoints[componentName] = make(map[string]string) + component.Iterate(func(key string, value interface{}) error { + displayDebugFuzzPoints[componentName][key] = fmt.Sprintf("%v", value) + return nil + }) + } finalComponentList = append(finalComponentList, component) } + if len(displayDebugFuzzPoints) > 0 { + gologger.Info().Msgf("Fuzz points for %s\n", rule.options.TemplateID) + marshalled, _ := json.MarshalIndent(displayDebugFuzzPoints, "", " ") + gologger.Silent().Msgf("%s\n", string(marshalled)) + } if len(finalComponentList) == 0 { return ErrRuleNotApplicable.Msgf("no component matched on this rule") diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index 4c01135f66..c69fbe06d6 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -18,7 +18,7 @@ func (rule *Rule) executePartRule(input *ExecuteRuleInput, payload ValueOrKeyVal // checkRuleApplicableOnComponent checks if a rule is applicable on given component func (rule *Rule) checkRuleApplicableOnComponent(component component.Component) bool { - if rule.Part != component.Name() { + if rule.Part != component.Name() && rule.partType != requestPartType { return false } foundAny := false diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index bc5f9caa9b..40f062d0dd 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -121,7 +121,8 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value } err := rule.Execute(&fuzz.ExecuteRuleInput{ - Input: input, + Input: input, + DisplayFuzzPoints: request.options.Options.DisplayFuzzPoints, Callback: func(gr fuzz.GeneratedRequest) bool { select { case <-input.Context().Done(): @@ -140,6 +141,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value continue } if fuzz.IsErrRuleNotApplicable(err) { + gologger.Verbose().Msgf("[%s] fuzz: rule not applicable : %s\n", request.options.TemplateID, err) continue } if err == types.ErrNoMoreRequests { diff --git a/pkg/testutils/fuzzplayground/server.go b/pkg/testutils/fuzzplayground/server.go index 04f00a99df..af42552cbf 100644 --- a/pkg/testutils/fuzzplayground/server.go +++ b/pkg/testutils/fuzzplayground/server.go @@ -27,6 +27,7 @@ func GetPlaygroundServer() *echo.Echo { e.GET("/request", requestHandler) e.GET("/email", emailHandler) e.GET("/permissions", permissionsHandler) + e.GET("/blog/post", numIdorHandler) // for num based idors like ?id=44 e.POST("/reset-password", resetPasswordHandler) e.GET("/host-header-lab", hostHeaderLabHandler) @@ -47,13 +48,20 @@ var bodyTemplate = ` func indexHandler(ctx echo.Context) error { return ctx.HTML(200, fmt.Sprintf(bodyTemplate, `

Fuzzing Playground

- + `)) } diff --git a/pkg/types/types.go b/pkg/types/types.go index d7dd41f22d..487582326f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -361,6 +361,8 @@ type Options struct { FuzzingMode string // TlsImpersonate enables TLS impersonation TlsImpersonate bool + // DisplayFuzzPoints enables display of fuzz points for fuzzing + DisplayFuzzPoints bool // CodeTemplateSignaturePublicKey is the custom public key used to verify the template signature (algorithm is automatically inferred from the length) CodeTemplateSignaturePublicKey string // CodeTemplateSignatureAlgorithm specifies the sign algorithm (rsa, ecdsa) From 9592ec2325fac485859d07dfa55fd3ecf0a3bbeb Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 6 May 2024 23:25:02 +0530 Subject: [PATCH 05/12] feat: added support for fuzzing nested path segments --- pkg/fuzz/component/path.go | 50 +++++++++++++++++++++++++-------- pkg/fuzz/component/path_test.go | 40 ++++++++++++++++++++++++-- pkg/fuzz/component/query.go | 1 + pkg/fuzz/execute.go | 5 ++-- pkg/fuzz/parts.go | 12 +++++--- 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index 9d2e758025..b9aebd61af 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -2,10 +2,12 @@ package component import ( "context" + "strconv" + "strings" - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" ) // Path is a component for a request Path @@ -31,13 +33,18 @@ func (q *Path) Name() string { // parsed component func (q *Path) Parse(req *retryablehttp.Request) (bool, error) { q.req = req - q.value = NewValue(req.URL.Path) + q.value = NewValue("") - parsed, err := dataformat.Get(dataformat.RawDataFormat).Decode(q.value.String()) - if err != nil { - return false, err + splitted := strings.Split(req.URL.Path, "/") + values := make(map[string]interface{}) + for i := range splitted { + pathTillNow := strings.Join(splitted[:i+1], "/") + if pathTillNow == "" { + continue + } + values[strconv.Itoa(i)] = pathTillNow } - q.value.SetParsed(parsed, dataformat.RawDataFormat) + q.value.SetParsed(dataformat.KVMap(values), "") return true, nil } @@ -56,7 +63,8 @@ func (q *Path) Iterate(callback func(key string, value interface{}) error) (err // SetValue sets a value in the component // for a key func (q *Path) SetValue(key string, value string) error { - if !q.value.SetParsedValue(key, value) { + escaped := urlutil.ParamEncode(value) + if !q.value.SetParsedValue(key, escaped) { return ErrSetValue } return nil @@ -73,13 +81,31 @@ func (q *Path) Delete(key string) error { // Rebuild returns a new request with the // component rebuilt func (q *Path) Rebuild() (*retryablehttp.Request, error) { - encoded, err := q.value.Encode() - if err != nil { - return nil, errors.Wrap(err, "could not encode query") + originalValues := make(map[string]interface{}) + splitted := strings.Split(q.req.URL.Path, "/") + for i := range splitted { + pathTillNow := strings.Join(splitted[:i+1], "/") + if pathTillNow == "" { + continue + } + originalValues[strconv.Itoa(i)] = pathTillNow + } + + originalPath := q.req.URL.Path + lengthSplitted := len(q.value.parsed.Map) + for i := lengthSplitted; i > 0; i-- { + key := strconv.Itoa(i) + original := originalValues[key].(string) + new := q.value.parsed.Map[key].(string) + originalPath = strings.Replace(originalPath, original, new, 1) } + + rebuiltPath := originalPath + + // Clone the request and update the path cloned := q.req.Clone(context.Background()) - if err := cloned.UpdateRelPath(encoded, true); err != nil { - cloned.URL.RawPath = encoded + if err := cloned.UpdateRelPath(rebuiltPath, true); err != nil { + cloned.URL.RawPath = rebuiltPath } return cloned, nil } diff --git a/pkg/fuzz/component/path_test.go b/pkg/fuzz/component/path_test.go index 859ffcde12..b3882b3218 100644 --- a/pkg/fuzz/component/path_test.go +++ b/pkg/fuzz/component/path_test.go @@ -28,10 +28,10 @@ func TestURLComponent(t *testing.T) { return nil }) - require.Equal(t, []string{"value"}, keys, "unexpected keys") + require.Equal(t, []string{"1"}, keys, "unexpected keys") require.Equal(t, []string{"/testpath"}, values, "unexpected values") - err = urlComponent.SetValue("value", "/newpath") + err = urlComponent.SetValue("1", "/newpath") if err != nil { t.Fatal(err) } @@ -40,7 +40,41 @@ func TestURLComponent(t *testing.T) { if err != nil { t.Fatal(err) } - require.Equal(t, "/newpath", rebuilt.URL.Path, "unexpected URL path") require.Equal(t, "https://example.com/newpath", rebuilt.URL.String(), "unexpected full URL") } + +func TestURLComponent_NestedPaths(t *testing.T) { + path := NewPath() + req, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com/user/753/profile", nil) + if err != nil { + t.Fatal(err) + } + found, err := path.Parse(req) + if err != nil { + t.Fatal(err) + } + if !found { + t.Fatal("expected path to be found") + } + + isSet := false + + path.Iterate(func(key string, value interface{}) error { + if !isSet && value.(string) == "/user/753" { + isSet = true + if setErr := path.SetValue(key, "/user/753'"); setErr != nil { + t.Fatal(setErr) + } + } + return nil + }) + + newReq, err := path.Rebuild() + if err != nil { + t.Fatal(err) + } + if newReq.URL.Path != "/user/753'/profile" { + t.Fatal("expected path to be modified") + } +} diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go index 3dc07e6cf1..571161ee13 100644 --- a/pkg/fuzz/component/query.go +++ b/pkg/fuzz/component/query.go @@ -61,6 +61,7 @@ func (q *Query) Iterate(callback func(key string, value interface{}) error) (err // SetValue sets a value in the component // for a key func (q *Query) SetValue(key string, value string) error { + // Is this safe? if !q.value.SetParsedValue(key, value) { return ErrSetValue } diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index f05b634225..f25f2f3482 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -109,9 +109,8 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { finalComponentList = append(finalComponentList, component) } if len(displayDebugFuzzPoints) > 0 { - gologger.Info().Msgf("Fuzz points for %s\n", rule.options.TemplateID) marshalled, _ := json.MarshalIndent(displayDebugFuzzPoints, "", " ") - gologger.Silent().Msgf("%s\n", string(marshalled)) + gologger.Info().Msgf("[%s] Fuzz points for %s [%s]\n%s\n", rule.options.TemplateID, input.Input.MetaInput.Input, input.BaseRequest.Method, string(marshalled)) } if len(finalComponentList) == 0 { @@ -243,7 +242,7 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo if err != nil { return err } - if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil { + if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", ""); gotErr != nil { return gotErr } } diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index c69fbe06d6..f2bae5d541 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -2,6 +2,7 @@ package fuzz import ( "io" + "strconv" "strings" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component" @@ -68,7 +69,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, valueStr); qerr != nil { return qerr } // fmt.Printf("executed with value: %s\n", evaluated) @@ -90,7 +91,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS if err != nil { return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, "", ""); qerr != nil { err = qerr return err } @@ -125,7 +126,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu return err } - if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil { + if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key, value); qerr != nil { return err } @@ -144,7 +145,10 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu } // execWithInput executes a rule with input via callback -func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error { +func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter, parameterValue string) error { + if _, err := strconv.Atoi(parameter); err == nil || (parameter == "" && parameterValue != "") { + parameter = parameterValue + } request := GeneratedRequest{ Request: httpReq, InteractURLs: interactURLs, From ddf2955f23c65fffcfb2ccb284ee939d5e5a0e24 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 14:22:00 +0530 Subject: [PATCH 06/12] feat: added parts to fuzzing requests --- pkg/fuzz/execute.go | 8 +++++--- pkg/fuzz/fuzz.go | 19 +++++++++++++++++-- pkg/fuzz/parts.go | 3 ++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index f25f2f3482..3ab84ea77d 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" + sliceutil "github.com/projectdiscovery/utils/slice" urlutil "github.com/projectdiscovery/utils/url" ) @@ -81,7 +82,7 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { // match rule part with component name displayDebugFuzzPoints := make(map[string]map[string]string) for _, componentName := range component.Components { - if !(rule.Part == componentName || rule.partType == requestPartType) { + if !(rule.Part == componentName || sliceutil.Contains(rule.Parts, componentName) || rule.partType == requestPartType) { continue } component := component.New(componentName) @@ -278,8 +279,9 @@ func (rule *Rule) Compile(generator *generators.PayloadGenerator, options *proto } else { rule.partType = valueType } - } else { - rule.partType = queryPartType + } + if rule.Part == "" && len(rule.Parts) == 0 { + return errors.Errorf("no part specified for rule") } if rule.Type != "" { diff --git a/pkg/fuzz/fuzz.go b/pkg/fuzz/fuzz.go index c70bc3b2cc..b2109efc85 100644 --- a/pkg/fuzz/fuzz.go +++ b/pkg/fuzz/fuzz.go @@ -24,12 +24,27 @@ type Rule struct { ruleType ruleType // description: | // Part is the part of request to fuzz. - // - // query fuzzes the query part of url. More parts will be added later. // values: // - "query" + // - "header" + // - "path" + // - "body" + // - "cookie" + // - "request" Part string `yaml:"part,omitempty" json:"part,omitempty" jsonschema:"title=part of rule,description=Part of request rule to fuzz,enum=query,enum=header,enum=path,enum=body,enum=cookie,enum=request"` partType partType + // description: | + // Parts is the list of parts to fuzz. If multiple parts need to be + // defined while excluding some, this should be used instead of singular part. + // values: + // - "query" + // - "header" + // - "path" + // - "body" + // - "cookie" + // - "request" + Parts []string `yaml:"parts,omitempty" json:"parts,omitempty" jsonschema:"title=parts of rule,description=Part of request rule to fuzz,enum=query,enum=header,enum=path,enum=body,enum=cookie,enum=request"` + // description: | // Mode is the mode of fuzzing to perform. // diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index f2bae5d541..d125c3ed2b 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryablehttp-go" + sliceutil "github.com/projectdiscovery/utils/slice" ) // executePartRule executes part rules based on type @@ -19,7 +20,7 @@ func (rule *Rule) executePartRule(input *ExecuteRuleInput, payload ValueOrKeyVal // checkRuleApplicableOnComponent checks if a rule is applicable on given component func (rule *Rule) checkRuleApplicableOnComponent(component component.Component) bool { - if rule.Part != component.Name() && rule.partType != requestPartType { + if rule.Part != component.Name() && !sliceutil.Contains(rule.Parts, component.Name()) && rule.partType != requestPartType { return false } foundAny := false From 63c5fb31c64730f171dc060c58575903e778553b Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 16:31:31 +0530 Subject: [PATCH 07/12] feat: added tracking for parameter occurence frequency in fuzzing --- internal/runner/runner.go | 67 ++++---- pkg/fuzz/frequency/tracker.go | 152 ++++++++++++++++++ pkg/fuzz/parts.go | 13 ++ pkg/protocols/common/interactsh/interactsh.go | 12 ++ pkg/protocols/common/interactsh/options.go | 6 +- pkg/protocols/http/request_fuzz.go | 11 ++ pkg/protocols/protocols.go | 3 + 7 files changed, 232 insertions(+), 32 deletions(-) create mode 100644 pkg/fuzz/frequency/tracker.go diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 43f16ff10b..7c39dc31cb 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/internal/pdcp" "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" "github.com/projectdiscovery/nuclei/v3/pkg/installer" "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" @@ -74,21 +75,22 @@ var ( // Runner is a client for running the enumeration process. type Runner struct { - output output.Writer - interactsh *interactsh.Client - options *types.Options - projectFile *projectfile.ProjectFile - catalog catalog.Catalog - progress progress.Progress - colorizer aurora.Aurora - issuesClient reporting.Client - browser *engine.Browser - rateLimiter *ratelimit.Limiter - hostErrors hosterrorscache.CacheInterface - resumeCfg *types.ResumeCfg - pprofServer *http.Server - pdcpUploadErrMsg string - inputProvider provider.InputProvider + output output.Writer + interactsh *interactsh.Client + options *types.Options + projectFile *projectfile.ProjectFile + catalog catalog.Catalog + progress progress.Progress + colorizer aurora.Aurora + issuesClient reporting.Client + browser *engine.Browser + rateLimiter *ratelimit.Limiter + hostErrors hosterrorscache.CacheInterface + resumeCfg *types.ResumeCfg + pprofServer *http.Server + pdcpUploadErrMsg string + inputProvider provider.InputProvider + fuzzFrequencyCache *frequency.Tracker //general purpose temporary directory tmpDir string parser parser.Parser @@ -440,24 +442,28 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) } + fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, frequency.DefaultParamOccurenceThreshold) + r.fuzzFrequencyCache = fuzzFreqCache + // Create the executor options which will be used throughout the execution // stage by the nuclei engine modules. executorOpts := protocols.ExecutorOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - IssuesClient: r.issuesClient, - RateLimiter: r.rateLimiter, - Interactsh: r.interactsh, - ProjectFile: r.projectFile, - Browser: r.browser, - Colorizer: r.colorizer, - ResumeCfg: r.resumeCfg, - ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers), - InputHelper: input.NewHelper(), - TemporaryDirectory: r.tmpDir, - Parser: r.parser, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + IssuesClient: r.issuesClient, + RateLimiter: r.rateLimiter, + Interactsh: r.interactsh, + ProjectFile: r.projectFile, + Browser: r.browser, + Colorizer: r.colorizer, + ResumeCfg: r.resumeCfg, + ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers), + InputHelper: input.NewHelper(), + TemporaryDirectory: r.tmpDir, + Parser: r.parser, + FuzzParamsFrequency: fuzzFreqCache, } if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" { @@ -600,6 +606,7 @@ func (r *Runner) RunEnumeration() error { if executorOpts.InputHelper != nil { _ = executorOpts.InputHelper.Close() } + r.fuzzFrequencyCache.Close() // todo: error propagation without canonical straight error check is required by cloud? // use safe dereferencing to avoid potential panics in case of previous unchecked errors diff --git a/pkg/fuzz/frequency/tracker.go b/pkg/fuzz/frequency/tracker.go new file mode 100644 index 0000000000..063912b4c0 --- /dev/null +++ b/pkg/fuzz/frequency/tracker.go @@ -0,0 +1,152 @@ +package frequency + +import ( + "net" + "net/url" + "os" + "strings" + "sync" + "sync/atomic" + + "github.com/bluele/gcache" + "github.com/projectdiscovery/gologger" +) + +// Tracker implements a frequency tracker for a given input +// which is used to determine uninteresting input parameters +// which are not that interesting from fuzzing perspective for a template +// and target combination. +// +// This is used to reduce the number of requests made during fuzzing +// for parameters that are less likely to give results for a rule. +type Tracker struct { + frequencies gcache.Cache + paramOccurenceThreshold int + + isDebug bool +} + +const ( + DefaultMaxTrackCount = 10000 + DefaultParamOccurenceThreshold = 10 +) + +type cacheItem struct { + errors atomic.Int32 + sync.Once +} + +// New creates a new frequency tracker with a given maximum +// number of params to track in LRU fashion with a max error threshold +func New(maxTrackCount, paramOccurenceThreshold int) *Tracker { + gc := gcache.New(maxTrackCount).ARC().Build() + + var isDebug bool + if os.Getenv("FREQ_DEBUG") != "" { + isDebug = true + } + return &Tracker{ + isDebug: isDebug, + frequencies: gc, + paramOccurenceThreshold: paramOccurenceThreshold, + } +} + +func (t *Tracker) Close() { + t.frequencies.Purge() +} + +// MarkParameter marks a parameter as frequently occuring once. +// +// The logic requires a parameter to be marked as frequently occuring +// multiple times before it's considered as frequently occuring. +func (t *Tracker) MarkParameter(parameter, target, template string) { + normalizedTarget := normalizeTarget(target) + key := getFrequencyKey(parameter, normalizedTarget, template) + + if t.isDebug { + gologger.Verbose().Msgf("[%s] Marking %s as found uninteresting", template, key) + } + + existingCacheItem, err := t.frequencies.GetIFPresent(key) + if err != nil || existingCacheItem == nil { + newItem := &cacheItem{errors: atomic.Int32{}} + newItem.errors.Store(1) + _ = t.frequencies.Set(key, newItem) + return + } + existingCacheItemValue := existingCacheItem.(*cacheItem) + existingCacheItemValue.errors.Add(1) + + _ = t.frequencies.Set(key, existingCacheItemValue) +} + +// IsParameterFrequent checks if a parameter is frequently occuring +// in the input with no much results. +func (t *Tracker) IsParameterFrequent(parameter, target, template string) bool { + normalizedTarget := normalizeTarget(target) + key := getFrequencyKey(parameter, normalizedTarget, template) + + if t.isDebug { + gologger.Verbose().Msgf("[%s] Checking if %s is frequently found uninteresting", template, key) + } + + existingCacheItem, err := t.frequencies.GetIFPresent(key) + if err != nil { + return false + } + existingCacheItemValue := existingCacheItem.(*cacheItem) + + if existingCacheItemValue.errors.Load() >= int32(t.paramOccurenceThreshold) { + existingCacheItemValue.Do(func() { + gologger.Verbose().Msgf("[%s] Skipped %s from parameter for %s as found uninteresting %d times", template, parameter, target, existingCacheItemValue.errors.Load()) + }) + return true + } + return false +} + +// UnmarkParameter unmarks a parameter as frequently occuring. This carries +// more weight and resets the frequency counter for the parameter causing +// it to be checked again. This is done when results are found. +func (t *Tracker) UnmarkParameter(parameter, target, template string) { + normalizedTarget := normalizeTarget(target) + key := getFrequencyKey(parameter, normalizedTarget, template) + + if t.isDebug { + gologger.Verbose().Msgf("[%s] Unmarking %s as frequently found uninteresting", template, key) + } + + _ = t.frequencies.Remove(key) +} + +func getFrequencyKey(parameter, target, template string) string { + var sb strings.Builder + sb.WriteString(target) + sb.WriteString(":") + sb.WriteString(template) + sb.WriteString(":") + sb.WriteString(parameter) + str := sb.String() + return str +} + +func normalizeTarget(value string) string { + finalValue := value + if strings.HasPrefix(value, "http") { + if parsed, err := url.Parse(value); err == nil { + hostname := parsed.Host + finalPort := parsed.Port() + if finalPort == "" { + if parsed.Scheme == "https" { + finalPort = "443" + } else { + finalPort = "80" + } + hostname = net.JoinHostPort(parsed.Host, finalPort) + } + finalValue = hostname + } + } + return finalValue +} diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index d125c3ed2b..6ab1643296 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -147,9 +147,22 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu // execWithInput executes a rule with input via callback func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter, parameterValue string) error { + // If the parameter is a number, replace it with the parameter value + // or if the parameter is empty and the parameter value is not empty + // replace it with the parameter value if _, err := strconv.Atoi(parameter); err == nil || (parameter == "" && parameterValue != "") { parameter = parameterValue } + // If the parameter is frequent, skip it if the option is enabled + if rule.options.FuzzParamsFrequency != nil { + if rule.options.FuzzParamsFrequency.IsParameterFrequent( + parameter, + httpReq.URL.String(), + rule.options.TemplateID, + ) { + return nil + } + } request := GeneratedRequest{ Request: httpReq, InteractURLs: interactURLs, diff --git a/pkg/protocols/common/interactsh/interactsh.go b/pkg/protocols/common/interactsh/interactsh.go index c156164805..da59f10fb3 100644 --- a/pkg/protocols/common/interactsh/interactsh.go +++ b/pkg/protocols/common/interactsh/interactsh.go @@ -21,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -180,6 +181,14 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d gologger.DefaultLogger.Print().Msgf("[Interactsh]: got result %v and status %v after processing interaction", result, matched) } + if c.options.FuzzParamsFrequency != nil { + if !matched { + c.options.FuzzParamsFrequency.MarkParameter(data.Parameter, data.Request.URL.String(), data.Operators.TemplateID) + } else { + c.options.FuzzParamsFrequency.UnmarkParameter(data.Parameter, data.Request.URL.String(), data.Operators.TemplateID) + } + } + // if we don't match, return if !matched || result == nil { return false @@ -320,6 +329,9 @@ type RequestData struct { Operators *operators.Operators MatchFunc operators.MatchFunc ExtractFunc operators.ExtractFunc + + Parameter string + Request *retryablehttp.Request } // RequestEvent is the event for a network request sent by nuclei. diff --git a/pkg/protocols/common/interactsh/options.go b/pkg/protocols/common/interactsh/options.go index c2ae250291..ca3dd459c9 100644 --- a/pkg/protocols/common/interactsh/options.go +++ b/pkg/protocols/common/interactsh/options.go @@ -4,6 +4,7 @@ import ( "time" "github.com/projectdiscovery/interactsh/pkg/client" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" @@ -46,8 +47,9 @@ type Options struct { // NoColor disables printing colors for matches NoColor bool - StopAtFirstMatch bool - HTTPClient *retryablehttp.Client + FuzzParamsFrequency *frequency.Tracker + StopAtFirstMatch bool + HTTPClient *retryablehttp.Client } // DefaultOptions returns the default options for interactsh client diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index 40f062d0dd..aa3953b2e6 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -179,6 +179,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, result.FuzzingPosition = gr.Component.Name() } + setInteractshCallback := false if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil { requestData := &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, @@ -186,7 +187,10 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, Operators: request.CompiledOperators, MatchFunc: request.Match, ExtractFunc: request.Extract, + Parameter: gr.Parameter, + Request: gr.Request, } + setInteractshCallback = true request.options.Interactsh.RequestEvent(gr.InteractURLs, requestData) gotMatches = request.options.Interactsh.AlreadyMatched(requestData) } else { @@ -196,6 +200,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, if event.OperatorsResult != nil { gotMatches = event.OperatorsResult.Matched } + if request.options.FuzzParamsFrequency != nil && !setInteractshCallback { + if !gotMatches { + request.options.FuzzParamsFrequency.MarkParameter(gr.Parameter, gr.Request.URL.String(), request.options.TemplateID) + } else { + request.options.FuzzParamsFrequency.UnmarkParameter(gr.Parameter, gr.Request.URL.String(), request.options.TemplateID) + } + } }, 0) // If a variable is unresolved, skip all further requests if errors.Is(requestErr, ErrMissingVars) { diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 6b4904c8d8..68881fec5c 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" "github.com/projectdiscovery/nuclei/v3/pkg/input" "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" @@ -92,6 +93,8 @@ type ExecutorOptions struct { ExcludeMatchers *excludematchers.ExcludeMatchers // InputHelper is a helper for input normalization InputHelper *input.Helper + // FuzzParamsFrequency is a cache for parameter frequency + FuzzParamsFrequency *frequency.Tracker Operators []*operators.Operators // only used by offlinehttp module From ef6557e46905be12dd5c93aa92b900eab18de887 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 16:46:47 +0530 Subject: [PATCH 08/12] added cli flag for fuzz frequency --- cmd/nuclei/main.go | 1 + internal/runner/runner.go | 2 +- pkg/types/types.go | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 4227202c84..4c766cc73f 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -320,6 +320,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), + flagSet.IntVar(&options.FuzzParamFrequency, "fuzz-param-frequency", 10, "frequency of uninteresting parameters for fuzzing before skipping"), ) flagSet.CreateGroup("uncover", "Uncover", diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 7c39dc31cb..ad1a60a06d 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -442,7 +442,7 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) } - fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, frequency.DefaultParamOccurenceThreshold) + fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, r.options.FuzzParamFrequency) r.fuzzFrequencyCache = fuzzFreqCache // Create the executor options which will be used throughout the execution diff --git a/pkg/types/types.go b/pkg/types/types.go index 487582326f..32529f0a08 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -363,6 +363,8 @@ type Options struct { TlsImpersonate bool // DisplayFuzzPoints enables display of fuzz points for fuzzing DisplayFuzzPoints bool + // FuzzParamFrequency is the frequency of fuzzing parameters + FuzzParamFrequency int // CodeTemplateSignaturePublicKey is the custom public key used to verify the template signature (algorithm is automatically inferred from the length) CodeTemplateSignaturePublicKey string // CodeTemplateSignatureAlgorithm specifies the sign algorithm (rsa, ecdsa) From 6e2ad5c95d396b527ec05dd7bcd35bf17d7dc47a Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 19:09:24 +0530 Subject: [PATCH 09/12] fixed broken tests --- pkg/fuzz/component/path_test.go | 2 +- pkg/fuzz/execute.go | 2 +- pkg/fuzz/fuzz_test.go | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/fuzz/component/path_test.go b/pkg/fuzz/component/path_test.go index b3882b3218..c47f81f4ff 100644 --- a/pkg/fuzz/component/path_test.go +++ b/pkg/fuzz/component/path_test.go @@ -60,7 +60,7 @@ func TestURLComponent_NestedPaths(t *testing.T) { isSet := false - path.Iterate(func(key string, value interface{}) error { + _ = path.Iterate(func(key string, value interface{}) error { if !isSet && value.(string) == "/user/753" { isSet = true if setErr := path.SetValue(key, "/user/753'"); setErr != nil { diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index 3ab84ea77d..4f33ba6ddc 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -102,7 +102,7 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { // Debugging display for fuzz points if input.DisplayFuzzPoints { displayDebugFuzzPoints[componentName] = make(map[string]string) - component.Iterate(func(key string, value interface{}) error { + _ = component.Iterate(func(key string, value interface{}) error { displayDebugFuzzPoints[componentName][key] = fmt.Sprintf("%v", value) return nil }) diff --git a/pkg/fuzz/fuzz_test.go b/pkg/fuzz/fuzz_test.go index dc0771259b..6ef2e39b08 100644 --- a/pkg/fuzz/fuzz_test.go +++ b/pkg/fuzz/fuzz_test.go @@ -7,7 +7,9 @@ import ( ) func TestRuleMatchKeyOrValue(t *testing.T) { - rule := &Rule{} + rule := &Rule{ + Part: "query", + } err := rule.Compile(nil, nil) require.NoError(t, err, "could not compile rule") @@ -15,7 +17,7 @@ func TestRuleMatchKeyOrValue(t *testing.T) { require.True(t, result, "could not get correct result") t.Run("key", func(t *testing.T) { - rule := &Rule{Keys: []string{"url"}} + rule := &Rule{Keys: []string{"url"}, Part: "query"} err := rule.Compile(nil, nil) require.NoError(t, err, "could not compile rule") @@ -25,7 +27,7 @@ func TestRuleMatchKeyOrValue(t *testing.T) { require.False(t, result, "could not get correct result") }) t.Run("value", func(t *testing.T) { - rule := &Rule{ValuesRegex: []string{`https?:\/\/?([-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b)*(\/[\/\d\w\.-]*)*(?:[\?])*(.+)*`}} + rule := &Rule{ValuesRegex: []string{`https?:\/\/?([-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b)*(\/[\/\d\w\.-]*)*(?:[\?])*(.+)*`}, Part: "query"} err := rule.Compile(nil, nil) require.NoError(t, err, "could not compile rule") From 7a62bbafcd355970a735eaa88e4775de17b66342 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 19:31:04 +0530 Subject: [PATCH 10/12] fixed path based sqli integration test --- integration_tests/fuzz/fuzz-path-sqli.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/integration_tests/fuzz/fuzz-path-sqli.yaml index e23098d931..74c8f35b02 100644 --- a/integration_tests/fuzz/fuzz-path-sqli.yaml +++ b/integration_tests/fuzz/fuzz-path-sqli.yaml @@ -25,11 +25,10 @@ http: fuzzing: - part: path - type: replace-regex + type: replace mode: single - replace-regex: '/(.*?/)([0-9]+)(/.*)?' fuzz: - - '/${1}${2}{{pathsqli}}${3}' + - '{{pathsqli}}' matchers: - type: status From 12d116b8cd15195f564728b113a240eef5fd3786 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 19:31:40 +0530 Subject: [PATCH 11/12] feat: added configurable fuzzing aggression level for payloads --- cmd/nuclei/main.go | 1 + pkg/protocols/common/generators/generators.go | 69 ++++++++++++++++++- .../common/generators/generators_test.go | 48 +++++++++++++ pkg/types/types.go | 2 + 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 4a6106a8d1..fb03825c2a 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -325,6 +325,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), flagSet.IntVar(&options.FuzzParamFrequency, "fuzz-param-frequency", 10, "frequency of uninteresting parameters for fuzzing before skipping"), + flagSet.StringVarP(&options.FuzzAggressionLevel, "fuzz-aggression", "fa", "low", "fuzzing aggression level controls payload count for fuzz (low, medium, high)"), ) flagSet.CreateGroup("uncover", "Uncover", diff --git a/pkg/protocols/common/generators/generators.go b/pkg/protocols/common/generators/generators.go index b99de24c0b..5711b445f5 100644 --- a/pkg/protocols/common/generators/generators.go +++ b/pkg/protocols/common/generators/generators.go @@ -25,8 +25,19 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st // Resolve payload paths if they are files. payloadsFinal := make(map[string]interface{}) - for name, payload := range payloads { - payloadsFinal[name] = payload + for payloadName, v := range payloads { + switch value := v.(type) { + case map[interface{}]interface{}: + values, err := parsePayloadsWithAggression(payloadName, value, opts.FuzzAggressionLevel) + if err != nil { + return nil, errors.Wrap(err, "could not parse payloads with aggression") + } + for k, v := range values { + payloadsFinal[k] = v + } + default: + payloadsFinal[payloadName] = v + } } generator := &PayloadGenerator{catalog: catalog, options: opts} @@ -57,6 +68,60 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st return generator, nil } +type aggressionLevelToPayloads struct { + Low []interface{} + Medium []interface{} + High []interface{} +} + +// parsePayloadsWithAggression parses the payloads with the aggression level +// +// Three agression are supported - +// - low +// - medium +// - high +// +// low is the default level. If medium is specified, all templates from +// low and medium are executed. Similarly with high, including all templates +// from low, medium, high. +func parsePayloadsWithAggression(name string, v map[interface{}]interface{}, agression string) (map[string]interface{}, error) { + payloadsLevels := &aggressionLevelToPayloads{} + + for k, v := range v { + if _, ok := v.([]interface{}); !ok { + return nil, errors.Errorf("only lists are supported for aggression levels payloads") + } + var ok bool + switch k { + case "low": + payloadsLevels.Low, ok = v.([]interface{}) + case "medium": + payloadsLevels.Medium, ok = v.([]interface{}) + case "high": + payloadsLevels.High, ok = v.([]interface{}) + default: + return nil, errors.Errorf("invalid aggression level %s specified for %s", k, name) + } + if !ok { + return nil, errors.Errorf("invalid aggression level %s specified for %s", k, name) + } + } + + payloads := make(map[string]interface{}) + switch agression { + case "low": + payloads[name] = payloadsLevels.Low + case "medium": + payloads[name] = append(payloadsLevels.Low, payloadsLevels.Medium...) + case "high": + payloads[name] = append(payloadsLevels.Low, payloadsLevels.Medium...) + payloads[name] = append(payloads[name].([]interface{}), payloadsLevels.High...) + default: + return nil, errors.Errorf("invalid aggression level %s specified for %s", agression, name) + } + return payloads, nil +} + // Iterator is a single instance of an iterator for a generator structure type Iterator struct { Type AttackType diff --git a/pkg/protocols/common/generators/generators_test.go b/pkg/protocols/common/generators/generators_test.go index a55e8e51ee..c478995525 100644 --- a/pkg/protocols/common/generators/generators_test.go +++ b/pkg/protocols/common/generators/generators_test.go @@ -1,9 +1,11 @@ package generators import ( + "strings" "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -90,3 +92,49 @@ func getOptions(allowLocalFileAccess bool) *types.Options { opts.AllowLocalFileAccess = allowLocalFileAccess return opts } + +func TestParsePayloadsWithAggression(t *testing.T) { + testPayload := `linux_path: + low: + - /etc/passwd + medium: + - ../etc/passwd + - ../../etc/passwd + high: + - ../../../etc/passwd + - ../../../../etc/passwd + - ../../../../../etc/passwd` + + var payloads map[string]interface{} + err := yaml.NewDecoder(strings.NewReader(testPayload)).Decode(&payloads) + require.Nil(t, err, "could not unmarshal yaml") + + aggressionsToValues := map[string][]string{ + "low": { + "/etc/passwd", + }, + "medium": { + "/etc/passwd", + "../etc/passwd", + "../../etc/passwd", + }, + "high": { + "/etc/passwd", + "../etc/passwd", + "../../etc/passwd", + "../../../etc/passwd", + "../../../../etc/passwd", + "../../../../../etc/passwd", + }, + } + + for k, v := range payloads { + for aggression, values := range aggressionsToValues { + parsed, err := parsePayloadsWithAggression(k, v.(map[interface{}]interface{}), aggression) + require.Nil(t, err, "could not parse payloads with aggression") + + gotValues := parsed[k].([]interface{}) + require.Equal(t, len(values), len(gotValues), "could not get correct number of values") + } + } +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 511da3e002..c2ff0c876a 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -365,6 +365,8 @@ type Options struct { TlsImpersonate bool // DisplayFuzzPoints enables display of fuzz points for fuzzing DisplayFuzzPoints bool + // FuzzAggressionLevel is the level of fuzzing aggression (low, medium, high.) + FuzzAggressionLevel string // FuzzParamFrequency is the frequency of fuzzing parameters FuzzParamFrequency int // CodeTemplateSignaturePublicKey is the custom public key used to verify the template signature (algorithm is automatically inferred from the length) From 57bede40209aad968986098703819403d64d2262 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 7 May 2024 21:45:59 +0530 Subject: [PATCH 12/12] fixed failing test --- integration_tests/fuzz/fuzz-path-sqli.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/integration_tests/fuzz/fuzz-path-sqli.yaml index 74c8f35b02..531427becb 100644 --- a/integration_tests/fuzz/fuzz-path-sqli.yaml +++ b/integration_tests/fuzz/fuzz-path-sqli.yaml @@ -15,17 +15,15 @@ http: - type: dsl dsl: - 'method == "GET"' - - regex("/(.*?/)([0-9]+)(/.*)?",path) condition: and payloads: pathsqli: - - "'OR1=1" - '%20OR%20True' fuzzing: - part: path - type: replace + type: postfix mode: single fuzz: - '{{pathsqli}}'