From 2418319df403b74ea2e38557adb1806589e9b87c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 27 Jul 2024 02:46:34 +0530 Subject: [PATCH] js: generate matcher-status event (#5450) * js: generate matcher-status event * isPortOpen: use fastdialer instance * update sdk unit test * add docs :) --- examples/advanced/advanced.go | 13 ++- pkg/js/compiler/compiler.go | 21 ++++- pkg/js/global/scripts.go | 26 ++++-- pkg/protocols/javascript/js.go | 150 +++++++++++++++++++-------------- 4 files changed, 137 insertions(+), 73 deletions(-) diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go index 110160f9a1..79355e5d9d 100644 --- a/examples/advanced/advanced.go +++ b/examples/advanced/advanced.go @@ -1,13 +1,24 @@ package main import ( + "context" + nuclei "github.com/projectdiscovery/nuclei/v3/lib" + "github.com/projectdiscovery/nuclei/v3/pkg/installer" syncutil "github.com/projectdiscovery/utils/sync" ) func main() { + ctx := context.Background() + // when running nuclei in parallel for first time it is a good practice to make sure + // templates exists first + tm := installer.TemplateManager{} + if err := tm.FreshInstallIfNotExists(); err != nil { + panic(err) + } + // create nuclei engine with options - ne, err := nuclei.NewThreadSafeNucleiEngine() + ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx) if err != nil { panic(err) } diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index f50e44ff2a..99cbcce923 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -55,6 +55,11 @@ type ExecuteArgs struct { TemplateCtx map[string]interface{} // templateCtx contains template scoped variables } +// Map returns a merged map of the TemplateCtx and Args fields. +func (e *ExecuteArgs) Map() map[string]interface{} { + return generators.MergeMaps(e.TemplateCtx, e.Args) +} + // NewExecuteArgs returns a new execute arguments. func NewExecuteArgs() *ExecuteArgs { return &ExecuteArgs{ @@ -66,12 +71,24 @@ func NewExecuteArgs() *ExecuteArgs { // ExecuteResult is the result of executing a script. type ExecuteResult map[string]interface{} +// Map returns the map representation of the ExecuteResult +func (e ExecuteResult) Map() map[string]interface{} { + if e == nil { + return make(map[string]interface{}) + } + return e +} + +// NewExecuteResult returns a new execute result instance func NewExecuteResult() ExecuteResult { return make(map[string]interface{}) } // GetSuccess returns whether the script was successful or not. func (e ExecuteResult) GetSuccess() bool { + if e == nil { + return false + } val, ok := e["success"].(bool) if !ok { return false @@ -114,7 +131,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, if val, ok := err.(*goja.Exception); ok { err = val.Unwrap() } - return nil, err + e := NewExecuteResult() + e["error"] = err.Error() + return e, err } var res ExecuteResult if opts.exports != nil { diff --git a/pkg/js/global/scripts.go b/pkg/js/global/scripts.go index c6771fadf3..2c1d56e12b 100644 --- a/pkg/js/global/scripts.go +++ b/pkg/js/global/scripts.go @@ -2,6 +2,7 @@ package global import ( "bytes" + "context" "embed" "math/rand" "net" @@ -12,8 +13,10 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -111,11 +114,16 @@ func initBuiltInFunc(runtime *goja.Runtime) { }, Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds", FuncDecl: func(host string, port string, timeout ...int) (bool, error) { - timeoutInSec := 5 + ctx := context.Background() if len(timeout) > 0 { - timeoutInSec = timeout[0] + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second) + defer cancel() } - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second) + if host == "" || port == "" { + return false, errkit.New("isPortOpen: host or port is empty") + } + conn, err := protocolstate.Dialer.Dial(ctx, "tcp", net.JoinHostPort(host, port)) if err != nil { return false, err } @@ -131,16 +139,20 @@ func initBuiltInFunc(runtime *goja.Runtime) { }, Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.", FuncDecl: func(host string, port string, timeout ...int) (bool, error) { - timeoutInSec := 5 + ctx := context.Background() if len(timeout) > 0 { - timeoutInSec = timeout[0] + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second) + defer cancel() + } + if host == "" || port == "" { + return false, errkit.New("isPortOpen: host or port is empty") } - conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second) + conn, err := protocolstate.Dialer.Dial(ctx, "udp", net.JoinHostPort(host, port)) if err != nil { return false, err } _ = conn.Close() - return true, nil }, }) diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 2e43c61cd9..fbcd1a6ff9 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -35,6 +35,7 @@ import ( "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" + mapsutil "github.com/projectdiscovery/utils/maps" syncutil "github.com/projectdiscovery/utils/sync" urlutil "github.com/projectdiscovery/utils/url" ) @@ -346,17 +347,33 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV TimeoutVariants: requestOptions.Options.GetTimeouts(), Source: &request.PreCondition, Context: target.Context(), }) - if err != nil { - return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) - } - if !result.GetSuccess() || types.ToString(result["error"]) != "" { - gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition) - request.options.Progress.IncrementFailedRequestsBy(1) - return nil - } - if request.options.Options.Debug || request.options.Options.DebugRequests { - request.options.Progress.IncrementRequests() - gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + // if precondition was successful + if err == nil && result.GetSuccess() { + if request.options.Options.Debug || request.options.Options.DebugRequests { + request.options.Progress.IncrementRequests() + gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + } + } else { + var outError error + // if js code failed to execute + if err != nil { + outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err) + } else { + // execution successful but pre-condition returned false + outError = errkit.New("pre-condition not satisfied skipping template execution") + } + results := map[string]interface{}(result) + results["error"] = outError.Error() + // generate and return failed event + data := request.generateEventData(input, results, hostPort) + data = generators.MergeMaps(data, payloadValues) + event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + allVars := argsCopy.Map() + allVars = generators.MergeMaps(allVars, data) + wrappedEvent.OperatorsResult.PayloadValues = allVars + }) + callback(event) + return err } } @@ -531,24 +548,72 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte } } + values := mapsutil.Merge(payloadValues, results) + // generate event data + data := request.generateEventData(input, values, hostPort) + + // add and get values from templatectx + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) + + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { + msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results)) + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { + gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg) + } + if requestOptions.Options.StoreResponse { + request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg) + } + } + + if _, ok := data["error"]; ok { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + wrappedEvent.OperatorsResult.PayloadValues = payload + }) + callback(event) + return err + } + + if request.options.Interactsh != nil { + request.options.Interactsh.MakePlaceholders(interactshURLs, data) + } + + var event *output.InternalWrappedEvent + if len(interactshURLs) == 0 { + event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + wrappedEvent.OperatorsResult.PayloadValues = payload + }) + callback(event) + } else if request.options.Interactsh != nil { + event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true} + request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, + Event: event, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, + }) + } + return nil +} + +// generateEventData generates event data for the request +func (request *Request) generateEventData(input *contextargs.Context, values map[string]interface{}, matched string) map[string]interface{} { data := make(map[string]interface{}) - for k, v := range payloadValues { + for k, v := range values { data[k] = v } data["type"] = request.Type().String() - for k, v := range results { - data[k] = v - } + data["request-pre-condition"] = beautifyJavascript(request.PreCondition) data["request"] = beautifyJavascript(request.Code) data["host"] = input.MetaInput.Input - data["matched"] = hostPort - data["template-path"] = requestOptions.TemplatePath - data["template-id"] = requestOptions.TemplateID - data["template-info"] = requestOptions.TemplateInfo + data["matched"] = matched + data["template-path"] = request.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo if request.StopAtFirstMatch || request.options.StopAtFirstMatch { data["stop-at-first-match"] = true } - // add ip address to data if input.MetaInput.CustomIP != "" { data["ip"] = input.MetaInput.CustomIP @@ -588,50 +653,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte } } } - - // add and get values from templatectx - request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data) - data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) - - if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped Javascript response for %s:\n%v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(results)) - if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { - gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg) - } - if requestOptions.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg) - } - } - - if _, ok := data["error"]; ok { - event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { - wrappedEvent.OperatorsResult.PayloadValues = payload - }) - callback(event) - return err - } - - if request.options.Interactsh != nil { - request.options.Interactsh.MakePlaceholders(interactshURLs, data) - } - - var event *output.InternalWrappedEvent - if len(interactshURLs) == 0 { - event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(data, payloadValues), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { - wrappedEvent.OperatorsResult.PayloadValues = payload - }) - callback(event) - } else if request.options.Interactsh != nil { - event = &output.InternalWrappedEvent{InternalEvent: data, UsesInteractsh: true} - request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{ - MakeResultFunc: request.MakeResultEvent, - Event: event, - Operators: request.CompiledOperators, - MatchFunc: request.Match, - ExtractFunc: request.Extract, - }) - } - return nil + return data } func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) {