Fixed issue with -ms option to scan non accessible host (projectdis…

…covery#5576)

* fail if OnResult callback is not called

* generate error message from error logs

* try..parse..

* fix lint

* add error message to last matcher event

* fix network protocol error logging

* log returned log from ExecuteWithResults

* add back specific logging

* clean up the msg

* minor

* init integration test for -ms

* add tests for http,network,js,ws protocols

* fix lint

* fix network test

* return err for dns protocol

* add integration test for dns protocol

cmd/integration-test/integration-test.go

@@ -55,6 +55,7 @@ var (
 		"dsl":             dslTestcases,
 		"flow":            flowTestcases,
 		"javascript":      jsTestcases,
+		"matcher-status":  matcherStatusTestcases,
 	}
 	// flakyTests are run with a retry count of 3
 	flakyTests = map[string]bool{

cmd/integration-test/matcher-status.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+)
+
+var matcherStatusTestcases = []TestCaseInfo{
+	{Path: "protocols/http/get.yaml", TestCase: &httpNoAccess{}},
+	{Path: "protocols/network/net-https.yaml", TestCase: &networkNoAccess{}},
+	{Path: "protocols/headless/headless-basic.yaml", TestCase: &headlessNoAccess{}},
+	{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNoAccess{}},
+	{Path: "protocols/websocket/basic.yaml", TestCase: &websocketNoAccess{}},
+	{Path: "protocols/dns/a.yaml", TestCase: &dnsNoAccess{}},
+}
+
+type httpNoAccess struct{}
+
+func (h *httpNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error != "no address found for host" {
+		return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got none")
+	}
+	return nil
+}
+
+type networkNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error != "no address found for host" {
+		return fmt.Errorf("unexpected result: expecting \"no address found for host\" error but got \"%s\"", event.Error)
+	}
+	return nil
+}
+
+type headlessNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *headlessNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-headless", "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error == "" {
+		return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+	}
+	return nil
+}
+
+type javascriptNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *javascriptNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error == "" {
+		return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+	}
+	return nil
+}
+
+type websocketNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *websocketNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "ws://trust_me_bro.real", debug, "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error == "" {
+		return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+	}
+	return nil
+}
+
+type dnsNoAccess struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *dnsNoAccess) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "trust_me_bro.real", debug, "-ms", "-j")
+	if err != nil {
+		return err
+	}
+	event := &output.ResultEvent{}
+	_ = json.Unmarshal([]byte(results[0]), event)
+
+	if event.Error == "" {
+		return fmt.Errorf("unexpected result: expecting an error but got \"%s\"", event.Error)
+	}
+	return nil
+}

pkg/protocols/dns/request.go

@@ -106,7 +106,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 }
 
 func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error {
-
+	var err error
 	if vardump.EnableVarDump {
 		gologger.Debug().Msgf("DNS Protocol request variables: \n%s\n", vardump.DumpVariables(vars))
 	}
@@ -199,7 +199,7 @@ func (request *Request) execute(input *contextargs.Context, domain string, metad
 	}
 
 	callback(event)
-	return nil
+	return err
 }
 
 func (request *Request) parseDNSInput(host string) (string, error) {

pkg/protocols/network/request.go

@@ -155,14 +155,14 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
 		}
 		visited.Set(actualAddress, struct{}{})
 
-		if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
+		if err = request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
 			outputEvent := request.responseToDSLMap("", "", "", address, "")
 			callback(&output.InternalWrappedEvent{InternalEvent: outputEvent})
 			gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err)
 			continue
 		}
 	}
-	return nil
+	return err
 }
 
 // executeAddress executes the request for an address

pkg/protocols/network/request_test.go

@@ -86,7 +86,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
-		require.Nil(t, err, "could not execute network request")
+		require.NotNil(t, err, "could not execute network request")
 	})
 	require.Nil(t, finalEvent.Results, "could not get event output from request")
 

pkg/scan/scan_context.go

@@ -52,6 +52,10 @@ func (s *ScanContext) Context() context.Context {
 	return s.ctx
 }
 
+func (s *ScanContext) GenerateErrorMessage() string {
+	return joinErrors(s.errors)
+}
+
 // GenerateResult returns final results slice from all events
 func (s *ScanContext) GenerateResult() []*output.ResultEvent {
 	s.m.Lock()
@@ -96,7 +100,7 @@ func (s *ScanContext) LogError(err error) {
 	}
 	s.errors = append(s.errors, err)
 
-	errorMessage := joinErrors(s.errors)
+	errorMessage := s.GenerateErrorMessage()
 
 	for _, result := range s.results {
 		result.Error = errorMessage

pkg/tmplexec/exec.go

@@ -10,6 +10,7 @@ import (
 	"github.com/dop251/goja"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@@ -19,6 +20,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/generic"
 	"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/multiproto"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
+	"github.com/projectdiscovery/utils/errkit"
 )
 
 // TemplateExecutor is an executor for a template
@@ -126,6 +129,8 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 	executed := &atomic.Bool{}
 	// matched in this case means something was exported / written to output
 	matched := &atomic.Bool{}
+	// callbackCalled tracks if the callback was called or not
+	callbackCalled := &atomic.Bool{}
 	defer func() {
 		// it is essential to remove template context of `Scan i.e template x input pair`
 		// since it is of no use after scan is completed (regardless of success or failure)
@@ -143,6 +148,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 	}
 
 	ctx.OnResult = func(event *output.InternalWrappedEvent) {
+		callbackCalled.Store(true)
 		if event == nil {
 			// something went wrong
 			return
@@ -198,13 +204,64 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 	} else {
 		errx = e.engine.ExecuteWithResults(ctx)
 	}
+	ctx.LogError(errx)
 
 	if lastMatcherEvent != nil {
+		lastMatcherEvent.InternalEvent["error"] = tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage()))
 		writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus)
 	}
+
+	//TODO: this is a hacky way to handle the case where the callback is not called and matcher-status is true.
+	// This is a workaround and needs to be refactored.
+	// Check if callback was never called and matcher-status is true
+	if !callbackCalled.Load() && e.options.Options.MatcherStatus {
+		fakeEvent := &output.InternalWrappedEvent{
+			Results: []*output.ResultEvent{
+				{
+					TemplateID: e.options.TemplateID,
+					Info:       e.options.TemplateInfo,
+					Type:       e.getTemplateType(),
+					Host:       ctx.Input.MetaInput.Input,
+					Error:      tryParseCause(fmt.Errorf("%s", ctx.GenerateErrorMessage())),
+				},
+			},
+			OperatorsResult: &operators.Result{
+				Matched: false,
+			},
+		}
+		writeFailureCallback(fakeEvent, e.options.Options.MatcherStatus)
+	}
+
 	return executed.Load() || matched.Load(), errx
 }
 
+// tryParseCause tries to parse the cause of given error
+// this is legacy support due to use of errorutil in existing libraries
+// but this should not be required once all libraries are updated
+func tryParseCause(err error) string {
+	errStr := ""
+	errX := errkit.FromError(err)
+	if errX != nil {
+		var errCause error
+
+		if len(errX.Errors()) > 1 {
+			errCause = errX.Errors()[0]
+		}
+		if errCause == nil {
+			errCause = errX
+		}
+
+		msg := strings.Trim(errCause.Error(), "{} ")
+		parts := strings.Split(msg, ":")
+		errCause = errkit.New("%s", parts[len(parts)-1])
+		errKind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic).String()
+		errStr = errCause.Error()
+		errStr = strings.TrimSpace(strings.Replace(errStr, "errKind="+errKind, "", -1))
+	}
+
+	return errStr
+}
+
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
 	var errx error