js: generate matcher-status event (projectdiscovery#5450)

* js: generate matcher-status event

* isPortOpen: use fastdialer instance

* update sdk unit test

* add docs :)

examples/advanced/advanced.go

@@ -1,13 +1,24 @@
 package main
 
 import (
+	"context"
+
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
+	ctx := context.Background()
+	// when running nuclei in parallel for first time it is a good practice to make sure
+	// templates exists first
+	tm := installer.TemplateManager{}
+	if err := tm.FreshInstallIfNotExists(); err != nil {
+		panic(err)
+	}
+
 	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx)
 	if err != nil {
 		panic(err)
 	}

pkg/js/compiler/compiler.go

@@ -55,6 +55,11 @@ type ExecuteArgs struct {
 	TemplateCtx map[string]interface{} // templateCtx contains template scoped variables
 }
 
+// Map returns a merged map of the TemplateCtx and Args fields.
+func (e *ExecuteArgs) Map() map[string]interface{} {
+	return generators.MergeMaps(e.TemplateCtx, e.Args)
+}
+
 // NewExecuteArgs returns a new execute arguments.
 func NewExecuteArgs() *ExecuteArgs {
 	return &ExecuteArgs{
@@ -66,12 +71,24 @@ func NewExecuteArgs() *ExecuteArgs {
 // ExecuteResult is the result of executing a script.
 type ExecuteResult map[string]interface{}
 
+// Map returns the map representation of the ExecuteResult
+func (e ExecuteResult) Map() map[string]interface{} {
+	if e == nil {
+		return make(map[string]interface{})
+	}
+	return e
+}
+
+// NewExecuteResult returns a new execute result instance
 func NewExecuteResult() ExecuteResult {
 	return make(map[string]interface{})
 }
 
 // GetSuccess returns whether the script was successful or not.
 func (e ExecuteResult) GetSuccess() bool {
+	if e == nil {
+		return false
+	}
 	val, ok := e["success"].(bool)
 	if !ok {
 		return false
@@ -114,7 +131,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 		if val, ok := err.(*goja.Exception); ok {
 			err = val.Unwrap()
 		}
-		return nil, err
+		e := NewExecuteResult()
+		e["error"] = err.Error()
+		return e, err
 	}
 	var res ExecuteResult
 	if opts.exports != nil {

pkg/js/global/scripts.go

@@ -2,6 +2,7 @@ package global
 
 import (
 	"bytes"
+	"context"
 	"embed"
 	"math/rand"
 	"net"
@@ -12,8 +13,10 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -111,11 +114,16 @@ func initBuiltInFunc(runtime *goja.Runtime) {
 		},
 		Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds",
 		FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
-			timeoutInSec := 5
+			ctx := context.Background()
 			if len(timeout) > 0 {
-				timeoutInSec = timeout[0]
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+				defer cancel()
 			}
-			conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+			if host == "" || port == "" {
+				return false, errkit.New("isPortOpen: host or port is empty")
+			}
+			conn, err := protocolstate.Dialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
 			if err != nil {
 				return false, err
 			}
@@ -131,16 +139,20 @@ func initBuiltInFunc(runtime *goja.Runtime) {
 		},
 		Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.",
 		FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
-			timeoutInSec := 5
+			ctx := context.Background()
 			if len(timeout) > 0 {
-				timeoutInSec = timeout[0]
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+				defer cancel()
+			}
+			if host == "" || port == "" {
+				return false, errkit.New("isPortOpen: host or port is empty")
 			}
-			conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+			conn, err := protocolstate.Dialer.Dial(ctx, "udp", net.JoinHostPort(host, port))
 			if err != nil {
 				return false, err
 			}
 			_ = conn.Close()
-
 			return true, nil
 		},
 	})

pkg/protocols/javascript/js.go

@@ -35,6 +35,7 @@ import (
 	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
@@ -346,17 +347,33 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				TimeoutVariants: requestOptions.Options.GetTimeouts(),
 				Source:          &request.PreCondition, Context: target.Context(),
 			})
-		if err != nil {
-			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
-		}
-		if !result.GetSuccess() || types.ToString(result["error"]) != "" {
-			gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition)
-			request.options.Progress.IncrementFailedRequestsBy(1)
-			return nil
-		}
-		if request.options.Options.Debug || request.options.Options.DebugRequests {
-			request.options.Progress.IncrementRequests()
-			gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
+		// if precondition was successful
+		if err == nil && result.GetSuccess() {
+			if request.options.Options.Debug || request.options.Options.DebugRequests {
+				request.options.Progress.IncrementRequests()
+				gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
+			}
+		} else {
+			var outError error
+			// if js code failed to execute
+			if err != nil {
+				outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err)
+			} else {
+				// execution successful but pre-condition returned false
+				outError = errkit.New("pre-condition not satisfied skipping template execution")
+			}
+			results := map[string]interface{}(result)
+			results["error"] = outError.Error()
+			// generate and return failed event
+			data := request.generateEventData(input, results, hostPort)
+			data = generators.MergeMaps(data, payloadValues)
+			event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+				allVars := argsCopy.Map()
+				allVars = generators.MergeMaps(allVars, data)
+				wrappedEvent.OperatorsResult.PayloadValues = allVars
+			})
+			callback(event)
+			return err
 		}
 	}
 
@@ -531,24 +548,72 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		}
 	}
 
+	values := mapsutil.Merge(payloadValues, results)
+	// generate event data
+	data := request.generateEventData(input, values, hostPort)
+
+	// add and get values from templatectx
+	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
+	data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
+
+	if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
+		msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results))
+		if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
+			gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
+		}
+		if requestOptions.Options.StoreResponse {
+			request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
+		}
+	}
+
+	if _, ok := data["error"]; ok {
+		event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+			wrappedEvent.OperatorsResult.PayloadValues = payload
+		})
+		callback(event)
+		return err
+	}
+
+	if request.options.Interactsh != nil {
+		request.options.Interactsh.MakePlaceholders(interactshURLs, data)
+	}
+
+	var event *output.InternalWrappedEvent
+	if len(interactshURLs) == 0 {
+		event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+			wrappedEvent.OperatorsResult.PayloadValues = payload
+		})
+		callback(event)
+	} else if request.options.Interactsh != nil {
+		event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true}
+		request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
+			MakeResultFunc: request.MakeResultEvent,
+			Event:          event,
+			Operators:      request.CompiledOperators,
+			MatchFunc:      request.Match,
+			ExtractFunc:    request.Extract,
+		})
+	}
+	return nil
+}
+
+// generateEventData generates event data for the request
+func (request *Request) generateEventData(input *contextargs.Context, values map[string]interface{}, matched string) map[string]interface{} {
 	data := make(map[string]interface{})
-	for k, v := range payloadValues {
+	for k, v := range values {
 		data[k] = v
 	}
 	data["type"] = request.Type().String()
-	for k, v := range results {
-		data[k] = v
-	}
+	data["request-pre-condition"] = beautifyJavascript(request.PreCondition)
 	data["request"] = beautifyJavascript(request.Code)
 	data["host"] = input.MetaInput.Input
-	data["matched"] = hostPort
-	data["template-path"] = requestOptions.TemplatePath
-	data["template-id"] = requestOptions.TemplateID
-	data["template-info"] = requestOptions.TemplateInfo
+	data["matched"] = matched
+	data["template-path"] = request.options.TemplatePath
+	data["template-id"] = request.options.TemplateID
+	data["template-info"] = request.options.TemplateInfo
 	if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
 		data["stop-at-first-match"] = true
 	}
-
 	// add ip address to data
 	if input.MetaInput.CustomIP != "" {
 		data["ip"] = input.MetaInput.CustomIP
@@ -588,50 +653,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 			}
 		}
 	}
-
-	// add and get values from templatectx
-	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
-	data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())
-
-	if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
-		msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results))
-		if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
-			gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
-		}
-		if requestOptions.Options.StoreResponse {
-			request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
-		}
-	}
-
-	if _, ok := data["error"]; ok {
-		event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
-			wrappedEvent.OperatorsResult.PayloadValues = payload
-		})
-		callback(event)
-		return err
-	}
-
-	if request.options.Interactsh != nil {
-		request.options.Interactsh.MakePlaceholders(interactshURLs, data)
-	}
-
-	var event *output.InternalWrappedEvent
-	if len(interactshURLs) == 0 {
-		event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
-			wrappedEvent.OperatorsResult.PayloadValues = payload
-		})
-		callback(event)
-	} else if request.options.Interactsh != nil {
-		event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true}
-		request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{
-			MakeResultFunc: request.MakeResultEvent,
-			Event:          event,
-			Operators:      request.CompiledOperators,
-			MatchFunc:      request.Match,
-			ExtractFunc:    request.Extract,
-		})
-	}
-	return nil
+	return data
 }
 
 func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) {