pre-condition in code , fuzz and other misc updates

.gitignore

@@ -35,4 +35,6 @@ pkg/protocols/headless/engine/.cache
 **/*-cache
 /fuzzplayground
 integration_tests/fuzzplayground
+/dsl.md
+
 

Makefile

@@ -47,4 +47,7 @@ fuzzplayground:
 memogen:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
 	./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl
-
+dsl-docs:
+	rm -f dsl.md scrapefuncs 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
+	./scrapefuncs -out dsl.md

cmd/integration-test/code.go

@@ -23,6 +23,7 @@ var codeTestCases = []TestCaseInfo{
 	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
 	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
 	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
+	{Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled},
 }
 
 const (
@@ -94,6 +95,22 @@ func (h *codeSnippet) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type codePreCondition struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *codePreCondition) Execute(filePath string) error {
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code")
+	if err != nil {
+		return err
+	}
+	if osutils.IsLinux() {
+		return expectResultsCount(results, 1)
+	} else {
+		return expectResultsCount(results, 0)
+
+	}
+}
+
 type codeFile struct{}
 
 // Execute executes a test case and returns an error if occurred

go.mod

@@ -97,7 +97,6 @@ require (
 	github.com/projectdiscovery/utils v0.0.84
 	github.com/projectdiscovery/wappalyzergo v0.0.112
 	github.com/redis/go-redis/v9 v9.1.0
-	github.com/sashabaranov/go-openai v1.15.3
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
@@ -205,6 +204,7 @@ require (
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
 	github.com/quic-go/quic-go v0.40.1 // indirect
 	github.com/refraction-networking/utls v1.6.1 // indirect
+	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect

integration_tests/flow/iterate-one-value-flow.yaml

@@ -4,10 +4,13 @@ info:
   name: Test Flow Iterate One Value Flow
   author: pdteam
   severity: info
+  description: |
+    If length of template.extracted variable is not know, i.e it could be an array of 1 or more values, then iterate function 
+    should be used to iterate over values because nuclei by default converts array to string if it has only 1 value.
 
 flow: |
   http(1)
-  for(let value of template.extracted){
+  for(let value of iterate(template.extracted)){
     set("value", value)
     http(2)
   }

integration_tests/fuzz/fuzz-body-generic-sqli.yaml

@@ -10,7 +10,7 @@ info:
     and performs fuzzing on the value of every key
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - method != "GET"

integration_tests/fuzz/fuzz-body-json-sqli.yaml

@@ -10,7 +10,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - method != "GET"

integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml

@@ -10,7 +10,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - method != "GET"

integration_tests/fuzz/fuzz-body-params-sqli.yaml

@@ -10,7 +10,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - method != "GET"

integration_tests/fuzz/fuzz-body-xml-sqli.yaml

@@ -10,7 +10,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - method != "GET"

integration_tests/fuzz/fuzz-cookie-error-sqli.yaml

@@ -9,7 +9,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - 'method == "GET"'

integration_tests/fuzz/fuzz-host-header-injection.yaml

@@ -10,7 +10,7 @@ variables:
   domain: "oast.fun"
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - 'method == "GET"'

integration_tests/fuzz/fuzz-path-sqli.yaml

@@ -11,7 +11,7 @@ info:
     Note: this is example template, and payloads/matchers need to be modified appropriately.
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - 'method == "GET"'

integration_tests/fuzz/fuzz-query-num-replace.yaml

@@ -7,7 +7,7 @@ info:
   description: Query Value Fuzzing using Fuzzing Rules
 
 http:
-  - filters:
+  - pre-condition:
       - type: dsl
         dsl:
           - 'len(query) > 0'
@@ -16,7 +16,7 @@ http:
         part: path
         words:
           - /blog/post
-    filters-condition: and
+    pre-condition-operator: and
 
     payloads:
       nums:

integration_tests/protocols/code/pre-condition.yaml

@@ -0,0 +1,26 @@
+id: pre-condition-code
+
+info:
+  name: example code template
+  author: pdteam
+  severity: info
+
+
+self-contained: true
+
+variables:
+  OAST: "{{interactsh-url}}"
+
+code:
+  - pre-condition: IsLinux()
+    engine:
+      - sh
+      - bash
+    source: |
+      echo "$OAST" | base64
+
+    matchers:
+      - type: dsl
+        dsl:
+          - true
+# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46

internal/runner/runner.go

@@ -688,14 +688,9 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 		}
 		for k, v := range templates.SignatureStats {
 			value := v.Load()
-			if k == templates.Unsigned && value > 0 {
-				// adjust skipped unsigned templates via code or -dut flag
-				value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats))
-				value = value - uint64(stats.GetValue(templates.ExcludedCodeTmplStats))
-			}
 			if value > 0 {
 				if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning {
-					gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value)
+					gologger.Print().Msgf("[%v] Executing %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value)
 				} else {
 					gologger.Info().Msgf("Executing %d signed templates from %s", value, k)
 				}

pkg/catalog/loader/loader.go

@@ -13,7 +13,6 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
-	cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -387,6 +386,21 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 	templatePathMap := store.pathFilter.Match(includedTemplates)
 
 	loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
+
+	loadTemplate := func(tmpl *templates.Template) {
+		loadedTemplates = append(loadedTemplates, tmpl)
+		// increment signed/unsigned counters
+		if tmpl.Verified {
+			if tmpl.TemplateVerifier == "" {
+				templates.SignatureStats[templates.PDVerifier].Add(1)
+			} else {
+				templates.SignatureStats[tmpl.TemplateVerifier].Add(1)
+			}
+		} else {
+			templates.SignatureStats[templates.Unsigned].Add(1)
+		}
+	}
+
 	for templatePath := range templatePathMap {
 		loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
 		if loaded || store.pathFilter.MatchIncluded(templatePath) {
@@ -412,7 +426,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 				if store.config.ExecutorOptions.Options.DAST {
 					// check if the template is a DAST template
 					if parsed.IsFuzzing() {
-						loadedTemplates = append(loadedTemplates, parsed)
+						loadTemplate(parsed)
 					}
 				} else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
 					// donot include headless template in final list if headless flag is not set
@@ -440,14 +454,14 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
 						gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
 					}
 				} else {
-					loadedTemplates = append(loadedTemplates, parsed)
+					loadTemplate(parsed)
 				}
 			}
 		}
 		if err != nil {
 			if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
 				stats.Increment(templates.TemplatesExcludedStats)
-				if cfg.DefaultConfig.LogAllEvents {
+				if config.DefaultConfig.LogAllEvents {
 					gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
 				}
 				continue

pkg/fuzz/dataformat/form.go

@@ -2,7 +2,6 @@ package dataformat
 
 import (
 	"fmt"
-	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
@@ -116,13 +115,11 @@ func (f *Form) Encode(data KV) (string, error) {
 
 // Decode decodes the data from Form format
 func (f *Form) Decode(data string) (KV, error) {
-	parsed, err := url.ParseQuery(data)
-	if err != nil {
-		return KV{}, err
-	}
+	ordered_params := urlutil.NewOrderedParams()
+	ordered_params.Merge(data)
 
 	values := mapsutil.NewOrderedMap[string, any]()
-	for key, value := range parsed {
+	ordered_params.Iterate(func(key string, value []string) bool {
 		if len(value) == 1 {
 			values.Set(key, value[0])
 		} else {
@@ -134,7 +131,8 @@ func (f *Form) Decode(data string) (KV, error) {
 			}
 			values.Set(key, value[len(value)-1])
 		}
-	}
+		return true
+	})
 	return KVOrderedMap(&values), nil
 }
 

pkg/fuzz/execute.go

@@ -64,11 +64,6 @@ type GeneratedRequest struct {
 // Input is not thread safe and should not be shared between concurrent
 // goroutines.
 func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("got panic while executing rule: %v", r)
-		}
-	}()
 	if !rule.isInputURLValid(input.Input) {
 		return ErrRuleNotApplicable.Msgf("invalid input url: %v", input.Input.MetaInput.Input)
 	}

pkg/input/types/http.go

@@ -207,11 +207,6 @@ func (hr *HttpResponse) Clone() *HttpResponse {
 // and returns the request and response object
 // Note: it currently does not parse response and is meant to be added manually since its a optional field
 func ParseRawRequest(raw string) (rr *RequestResponse, err error) {
-	defer func() {
-		if r := recover(); r != nil {
-			err = fmt.Errorf("panic: %v", r)
-		}
-	}()
 	protoReader := textproto.NewReader(bufio.NewReader(strings.NewReader(raw)))
 	methodLine, err := protoReader.ReadLine()
 	if err != nil {