js: generate matcher-status event (#5450)

* js: generate matcher-status event

* isPortOpen: use fastdialer instance

* update sdk unit test

* add docs :)

examples/advanced/advanced.go

@@ -1,13 +1,24 @@
 package main
 
 import (
+	"context"
+
 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
 func main() {
+	ctx := context.Background()
+	// when running nuclei in parallel for first time it is a good practice to make sure
+	// templates exists first
+	tm := installer.TemplateManager{}
+	if err := tm.FreshInstallIfNotExists(); err != nil {
+		panic(err)
+	}
+
 	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
+	ne, err := nuclei.NewThreadSafeNucleiEngineCtx(ctx)
 	if err != nil {
 		panic(err)
 	}

pkg/js/compiler/compiler.go

@@ -55,6 +55,11 @@ type ExecuteArgs struct {
 	TemplateCtx map[string]interface{} // templateCtx contains template scoped variables
 }
 
+// Map returns a merged map of the TemplateCtx and Args fields.
+func (e *ExecuteArgs) Map() map[string]interface{} {
+	return generators.MergeMaps(e.TemplateCtx, e.Args)
+}
+
 // NewExecuteArgs returns a new execute arguments.
 func NewExecuteArgs() *ExecuteArgs {
 	return &ExecuteArgs{
@@ -66,12 +71,24 @@ func NewExecuteArgs() *ExecuteArgs {
 // ExecuteResult is the result of executing a script.
 type ExecuteResult map[string]interface{}
 
+// Map returns the map representation of the ExecuteResult
+func (e ExecuteResult) Map() map[string]interface{} {
+	if e == nil {
+		return make(map[string]interface{})
+	}
+	return e
+}
+
+// NewExecuteResult returns a new execute result instance
 func NewExecuteResult() ExecuteResult {
 	return make(map[string]interface{})
 }
 
 // GetSuccess returns whether the script was successful or not.
 func (e ExecuteResult) GetSuccess() bool {
+	if e == nil {
+		return false
+	}
 	val, ok := e["success"].(bool)
 	if !ok {
 		return false
@@ -114,7 +131,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 		if val, ok := err.(*goja.Exception); ok {
 			err = val.Unwrap()
 		}
-		return nil, err
+		e := NewExecuteResult()
+		e["error"] = err.Error()
+		return e, err
 	}
 	var res ExecuteResult
 	if opts.exports != nil {

pkg/js/global/scripts.go

@@ -2,6 +2,7 @@ package global
 
 import (
 	"bytes"
+	"context"
 	"embed"
 	"math/rand"
 	"net"
@@ -12,8 +13,10 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -111,11 +114,16 @@ func initBuiltInFunc(runtime *goja.Runtime) {
 		},
 		Description: "isPortOpen checks if given TCP port is open on host. timeout is optional and defaults to 5 seconds",
 		FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
-			timeoutInSec := 5
+			ctx := context.Background()
 			if len(timeout) > 0 {
-				timeoutInSec = timeout[0]
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+				defer cancel()
 			}
-			conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+			if host == "" || port == "" {
+				return false, errkit.New("isPortOpen: host or port is empty")
+			}
+			conn, err := protocolstate.Dialer.Dial(ctx, "tcp", net.JoinHostPort(host, port))
 			if err != nil {
 				return false, err
 			}
@@ -131,16 +139,20 @@ func initBuiltInFunc(runtime *goja.Runtime) {
 		},
 		Description: "isUDPPortOpen checks if the given UDP port is open on the host. Timeout is optional and defaults to 5 seconds.",
 		FuncDecl: func(host string, port string, timeout ...int) (bool, error) {
-			timeoutInSec := 5
+			ctx := context.Background()
 			if len(timeout) > 0 {
-				timeoutInSec = timeout[0]
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout[0])*time.Second)
+				defer cancel()
 			}
-			conn, err := net.DialTimeout("udp", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)
+			if host == "" || port == "" {
+				return false, errkit.New("isPortOpen: host or port is empty")
+			}
+			conn, err := protocolstate.Dialer.Dial(ctx, "udp", net.JoinHostPort(host, port))
 			if err != nil {
 				return false, err
 			}
 			_ = conn.Close()
-
 			return true, nil
 		},
 	})

pkg/protocols/javascript/js.go

@@ -35,6 +35,7 @@ import (
 	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 	syncutil "github.com/projectdiscovery/utils/sync"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
@@ -346,17 +347,33 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				TimeoutVariants: requestOptions.Options.GetTimeouts(),
 				Source:          &request.PreCondition, Context: target.Context(),
 			})
-		if err != nil {
+		// if precondition was successful
-			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
+		if err == nil && result.GetSuccess() {
-		}
+			if request.options.Options.Debug || request.options.Options.DebugRequests {
-		if !result.GetSuccess() || types.ToString(result["error"]) != "" {
+				request.options.Progress.IncrementRequests()
-			gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition)
+				gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
-			request.options.Progress.IncrementFailedRequestsBy(1)
+			}
-			return nil
+		} else {
-		}
+			var outError error
-		if request.options.Options.Debug || request.options.Options.DebugRequests {
+			// if js code failed to execute
-			request.options.Progress.IncrementRequests()
+			if err != nil {
-			gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
+				outError = errkit.Append(errkit.New("pre-condition not satisfied skipping template execution"), err)
+			} else {
+				// execution successful but pre-condition returned false
+				outError = errkit.New("pre-condition not satisfied skipping template execution")
+			}
+			results := map[string]interface{}(result)
+			results["error"] = outError.Error()
+			// generate and return failed event
+			data := request.generateEventData(input, results, hostPort)
+			data = generators.MergeMaps(data, payloadValues)
+			event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
+				allVars := argsCopy.Map()
+				allVars = generators.MergeMaps(allVars, data)
+				wrappedEvent.OperatorsResult.PayloadValues = allVars
+			})
+			callback(event)
+			return err
 		}
 	}
 
@@ -531,63 +548,9 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 		}
 	}
 
-	data := make(map[string]interface{})
+	values := mapsutil.Merge(payloadValues, results)
-	for k, v := range payloadValues {
+	// generate event data
-		data[k] = v
+	data := request.generateEventData(input, values, hostPort)
-	}
-	data["type"] = request.Type().String()
-	for k, v := range results {
-		data[k] = v
-	}
-	data["request"] = beautifyJavascript(request.Code)
-	data["host"] = input.MetaInput.Input
-	data["matched"] = hostPort
-	data["template-path"] = requestOptions.TemplatePath
-	data["template-id"] = requestOptions.TemplateID
-	data["template-info"] = requestOptions.TemplateInfo
-	if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
-		data["stop-at-first-match"] = true
-	}
-
-	// add ip address to data
-	if input.MetaInput.CustomIP != "" {
-		data["ip"] = input.MetaInput.CustomIP
-	} else {
-		// context: https://github.com/projectdiscovery/nuclei/issues/5021
-		hostname := input.MetaInput.Input
-		if strings.Contains(hostname, ":") {
-			host, _, err := net.SplitHostPort(hostname)
-			if err == nil {
-				hostname = host
-			} else {
-				// naive way
-				if !strings.Contains(hostname, "]") {
-					hostname = hostname[:strings.LastIndex(hostname, ":")]
-				}
-			}
-		}
-		data["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
-		// if input itself was an ip, use it
-		if iputil.IsIP(hostname) {
-			data["ip"] = hostname
-		}
-
-		// if ip is not found,this is because ssh and other protocols do not use fastdialer
-		// although its not perfect due to its use case dial and get ip
-		dnsData, err := protocolstate.Dialer.GetDNSData(hostname)
-		if err == nil {
-			for _, v := range dnsData.A {
-				data["ip"] = v
-				break
-			}
-			if data["ip"] == "" {
-				for _, v := range dnsData.AAAA {
-					data["ip"] = v
-					break
-				}
-			}
-		}
-	}
 
 	// add and get values from templatectx
 	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), data)
@@ -634,6 +597,65 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 	return nil
 }
 
+// generateEventData generates event data for the request
+func (request *Request) generateEventData(input *contextargs.Context, values map[string]interface{}, matched string) map[string]interface{} {
+	data := make(map[string]interface{})
+	for k, v := range values {
+		data[k] = v
+	}
+	data["type"] = request.Type().String()
+	data["request-pre-condition"] = beautifyJavascript(request.PreCondition)
+	data["request"] = beautifyJavascript(request.Code)
+	data["host"] = input.MetaInput.Input
+	data["matched"] = matched
+	data["template-path"] = request.options.TemplatePath
+	data["template-id"] = request.options.TemplateID
+	data["template-info"] = request.options.TemplateInfo
+	if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
+		data["stop-at-first-match"] = true
+	}
+	// add ip address to data
+	if input.MetaInput.CustomIP != "" {
+		data["ip"] = input.MetaInput.CustomIP
+	} else {
+		// context: https://github.com/projectdiscovery/nuclei/issues/5021
+		hostname := input.MetaInput.Input
+		if strings.Contains(hostname, ":") {
+			host, _, err := net.SplitHostPort(hostname)
+			if err == nil {
+				hostname = host
+			} else {
+				// naive way
+				if !strings.Contains(hostname, "]") {
+					hostname = hostname[:strings.LastIndex(hostname, ":")]
+				}
+			}
+		}
+		data["ip"] = protocolstate.Dialer.GetDialedIP(hostname)
+		// if input itself was an ip, use it
+		if iputil.IsIP(hostname) {
+			data["ip"] = hostname
+		}
+
+		// if ip is not found,this is because ssh and other protocols do not use fastdialer
+		// although its not perfect due to its use case dial and get ip
+		dnsData, err := protocolstate.Dialer.GetDNSData(hostname)
+		if err == nil {
+			for _, v := range dnsData.A {
+				data["ip"] = v
+				break
+			}
+			if data["ip"] == "" {
+				for _, v := range dnsData.AAAA {
+					data["ip"] = v
+					break
+				}
+			}
+		}
+	}
+	return data
+}
+
 func (request *Request) getArgsCopy(input *contextargs.Context, payloadValues map[string]interface{}, requestOptions *protocols.ExecutorOptions, ignoreErrors bool) (*compiler.ExecuteArgs, error) {
 	// Template args from payloads
 	argsCopy, err := request.evaluateArgs(payloadValues, requestOptions, ignoreErrors)