From 2418319df403b74ea2e38557adb1806589e9b87c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Sat, 27 Jul 2024 02:46:34 +0530 Subject: [PATCH] js: generate matcher-status event (#5450) * js: generate matcher-status event * isPortOpen: use fastdialer instance * update sdk unit test * add docs :) --- examples/advanced/advanced.go | 13 ++- pkg/js/compiler/compiler.go | 21 ++++- pkg/js/global/scripts.go | 26 ++++-- pkg/protocols/javascript/js.go | 158 +++++++++++++++++++-------------- 4 files changed, 141 insertions(+), 77 deletions(-) diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go index 110160f9a..79355e5d9 100644 --- a/examples/advanced/advanced.go +++ b/examples/advanced/advanced.go @@ -1,13 +1,24 @@ package main import ( + "context" + nuclei "github.com/projectdiscovery/nuclei/v3/lib" + "github.com/projectdiscovery/nuclei/v3/pkg/installer" syncutil "github.com/projectdiscovery/utils/sync" ) func main() { + ctx := context.Background() + // when running nuclei in parallel for first time it is a good practice to make sure + // templates exists first + tm := installer.TemplateManager{} + if err := tm.FreshInstallIfNotExists(); err != nil { + panic(err) + } + // create nuclei engine with options - ne, err := nuclei.NewThreadSafeNucleiEngine() + ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx) if err != nil { panic(err) } diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index f50e44ff2..99cbcce92 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -55,6 +55,11 @@ type ExecuteArgs struct { TemplateCtx map[string]interface{} // templateCtx contains template scoped variables } +// Map returns a merged map of the TemplateCtx and Args fields. +func (e *ExecuteArgs) Map() map[string]interface{} { + return generators.MergeMaps(e.TemplateCtx, e.Args) +} + // NewExecuteArgs returns a new execute arguments. func NewExecuteArgs() *ExecuteArgs { return &ExecuteArgs{ @@ -66,12 +71,24 @@ func NewExecuteArgs() *ExecuteArgs { // ExecuteResult is the result of executing a script. type ExecuteResult map[string]interface{} +// Map returns the map representation of the ExecuteResult +func (e ExecuteResult) Map() map[string]interface{} { + if e == nil { + return make(map[string]interface{}) + } + return e +} + +// NewExecuteResult returns a new execute result instance func NewExecuteResult() ExecuteResult { return make(map[string]interface{}) } // GetSuccess returns whether the script was successful or not. func (e ExecuteResult) GetSuccess() bool { + if e == nil { + return false + } val, ok := e["success"].(bool) if !ok { return false @@ -114,7 +131,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, if val, ok := err.(*goja.Exception); ok { err = val.Unwrap() } - return nil, err + e := NewExecuteResult() + e["error"] = err.Error() + return e, err } var res ExecuteResult if opts.exports != nil { diff --git a/pkg/js/global/scripts.go b/pkg/js/global/scripts.go index c6771fadf..2c1d56e12 100644 --- a/pkg/js/global/scripts.go +++ b/pkg/js/global/scripts.go @@ -2,6 +2,7 @@ package global import ( "bytes" + "context" "embed" "math/rand" "net" @@ -12,8 +13,10 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -111,11 +114,16 @@ func initBuiltInFunc(runtime *goja.Runtime) { }, Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds", FuncDecl: func(host string, port string, timeout ...int) (bool, error) { - timeoutInSec := 5 + ctx := context.Background() if len(timeout) > 0 { - timeoutInSec = timeout[0] + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second) + defer cancel() } - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second) + if host == "" || port == "" { + return false, errkit.New("isPortOpen: host or port is empty") + } + conn, err := protocolstate.Dialer.Dial(ctx, "tcp", net.JoinHostPort(host, port)) if err != nil { return false, err } @@ -131,16 +139,20 @@ func initBuiltInFunc(runtime *goja.Runtime) { }, Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.", FuncDecl: func(host string, port string, timeout ...int) (bool, error) { - timeoutInSec := 5 + ctx := context.Background() if len(timeout) > 0 { - timeoutInSec = timeout[0] + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second) + defer cancel() } - conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second) + if host == "" || port == "" { + return false, errkit.New("isPortOpen: host or port is empty") + } + conn, err := protocolstate.Dialer.Dial(ctx, "udp", net.JoinHostPort(host, port)) if err != nil { return false, err } _ = conn.Close() - return true, nil }, }) diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 2e43c61cd..fbcd1a6ff 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -35,6 +35,7 @@ import ( "github.com/projectdiscovery/utils/errkit" errorutil "github.com/projectdiscovery/utils/errors" iputil "github.com/projectdiscovery/utils/ip" + mapsutil "github.com/projectdiscovery/utils/maps" syncutil "github.com/projectdiscovery/utils/sync" urlutil "github.com/projectdiscovery/utils/url" ) @@ -346,17 +347,33 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV TimeoutVariants: requestOptions.Options.GetTimeouts(), Source: &request.PreCondition, Context: target.Context(), }) - if err != nil { - return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) - } - if !result.GetSuccess() || types.ToString(result["error"]) != "" { - gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition) - request.options.Progress.IncrementFailedRequestsBy(1) - return nil - } - if request.options.Options.Debug || request.options.Options.DebugRequests { - request.options.Progress.IncrementRequests() - gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + // if precondition was successful + if err == nil && result.GetSuccess() { + if request.options.Options.Debug || request.options.Options.DebugRequests { + request.options.Progress.IncrementRequests() + gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + } + } else { + var outError error + // if js code failed to execute + if err != nil { + outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err) + } else { + // execution successful but pre-condition returned false + outError = errkit.New("pre-condition not satisfied skipping template execution") + } + results := map[string]interface{}(result) + results["error"] = outError.Error() + // generate and return failed event + data := request.generateEventData(input, results, hostPort) + data = generators.MergeMaps(data, payloadValues) + event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + allVars := argsCopy.Map() + allVars = generators.MergeMaps(allVars, data) + wrappedEvent.OperatorsResult.PayloadValues = allVars + }) + callback(event) + return err } } @@ -531,63 +548,9 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte } } - data := make(map[string]interface{}) - for k, v := range payloadValues { - data[k] = v - } - data["type"] = request.Type().String() - for k, v := range results { - data[k] = v - } - data["request"] = beautifyJavascript(request.Code) - data["host"] = input.MetaInput.Input - data["matched"] = hostPort - data["template-path"] = requestOptions.TemplatePath - data["template-id"] = requestOptions.TemplateID - data["template-info"] = requestOptions.TemplateInfo - if request.StopAtFirstMatch || request.options.StopAtFirstMatch { - data["stop-at-first-match"] = true - } - - // add ip address to data - if input.MetaInput.CustomIP != "" { - data["ip"] = input.MetaInput.CustomIP - } else { - // context: https://github.com/projectdiscovery/nuclei/issues/5021 - hostname := input.MetaInput.Input - if strings.Contains(hostname, ":") { - host, _, err := net.SplitHostPort(hostname) - if err == nil { - hostname = host - } else { - // naive way - if !strings.Contains(hostname, "]") { - hostname = hostname[:strings.LastIndex(hostname, ":")] - } - } - } - data["ip"] = protocolstate.Dialer.GetDialedIP(hostname) - // if input itself was an ip, use it - if iputil.IsIP(hostname) { - data["ip"] = hostname - } - - // if ip is not found,this is because ssh and other protocols do not use fastdialer - // although its not perfect due to its use case dial and get ip - dnsData, err := protocolstate.Dialer.GetDNSData(hostname) - if err == nil { - for _, v := range dnsData.A { - data["ip"] = v - break - } - if data["ip"] == "" { - for _, v := range dnsData.AAAA { - data["ip"] = v - break - } - } - } - } + values := mapsutil.Merge(payloadValues, results) + // generate event data + data := request.generateEventData(input, values, hostPort) // add and get values from templatectx request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data) @@ -634,6 +597,65 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte return nil } +// generateEventData generates event data for the request +func (request *Request) generateEventData(input *contextargs.Context, values map[string]interface{}, matched string) map[string]interface{} { + data := make(map[string]interface{}) + for k, v := range values { + data[k] = v + } + data["type"] = request.Type().String() + data["request-pre-condition"] = beautifyJavascript(request.PreCondition) + data["request"] = beautifyJavascript(request.Code) + data["host"] = input.MetaInput.Input + data["matched"] = matched + data["template-path"] = request.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + if request.StopAtFirstMatch || request.options.StopAtFirstMatch { + data["stop-at-first-match"] = true + } + // add ip address to data + if input.MetaInput.CustomIP != "" { + data["ip"] = input.MetaInput.CustomIP + } else { + // context: https://github.com/projectdiscovery/nuclei/issues/5021 + hostname := input.MetaInput.Input + if strings.Contains(hostname, ":") { + host, _, err := net.SplitHostPort(hostname) + if err == nil { + hostname = host + } else { + // naive way + if !strings.Contains(hostname, "]") { + hostname = hostname[:strings.LastIndex(hostname, ":")] + } + } + } + data["ip"] = protocolstate.Dialer.GetDialedIP(hostname) + // if input itself was an ip, use it + if iputil.IsIP(hostname) { + data["ip"] = hostname + } + + // if ip is not found,this is because ssh and other protocols do not use fastdialer + // although its not perfect due to its use case dial and get ip + dnsData, err := protocolstate.Dialer.GetDNSData(hostname) + if err == nil { + for _, v := range dnsData.A { + data["ip"] = v + break + } + if data["ip"] == "" { + for _, v := range dnsData.AAAA { + data["ip"] = v + break + } + } + } + } + return data +} + func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) { // Template args from payloads argsCopy, err := request.evaluateArgs(payloadValues, requestOptions, ignoreErrors)