bug fixes in js , network protocol and flow (#4313)

* fix net read

* only return N bytes if extra available

* use ConnReadN from readerutil

* add integration test

* print unsigned warning in stderr

* fix js protocol in flow #4318

* fix integration test: url encoding issue

* fix network protocol issue + integration tests

* multiple improvements to integration test

* replace all conn.Read() from tests

* disable network-basic.yaml in windows

* disable code protocol in win CI

* fix bitwise login  ps1-snippet.yaml

* hide previous matcher events in flow

* remove dead code+ update integration tests

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>

cmd/integration-test/code.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"log"
+	"os"
 	"path/filepath"
 
 	osutils "github.com/projectdiscovery/utils/os"
@@ -12,14 +13,16 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 )
 
+var isCodeDisabled = func() bool { return osutils.IsWindows() && os.Getenv("CI") == "true" }
+
 var codeTestCases = []TestCaseInfo{
-	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}},
+	{Path: "protocols/code/py-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}},
+	{Path: "protocols/code/py-file.yaml", TestCase: &codeFile{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}},
+	{Path: "protocols/code/py-env-var.yaml", TestCase: &codeEnvVar{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}},
+	{Path: "protocols/code/unsigned.yaml", TestCase: &unsignedCode{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}},
+	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}},
+	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
-	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() }},
+	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
 }
 
 const (
@@ -30,6 +33,10 @@ const (
 var testcertpath = ""
 
 func init() {
+	if isCodeDisabled() {
+		// skip executing code protocol in CI on windows
+		return
+	}
 	// allow local file access to load content of file references in template
 	// in order to sign them for testing purposes
 	templates.TemplateSignerLFA()

cmd/integration-test/flow.go

@@ -15,6 +15,7 @@ var flowTestcases = []TestCaseInfo{
 	{Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}},
 	{Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}},
 	{Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}},
+	{Path: "flow/flow-hide-matcher.yaml", TestCase: &flowHideMatcher{}},
 }
 
 type conditionalFlow struct{}
@@ -24,7 +25,7 @@ func (t *conditionalFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type conditionalFlowNegative struct{}
@@ -66,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 2)
+	return expectResultsCount(results, 1)
 }
 
 type dnsNsProbe struct{}
@@ -76,9 +77,20 @@ func (t *dnsNsProbe) Execute(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return expectResultsCount(results, 3)
+	return expectResultsCount(results, 1)
 }
 
 func getBase64(input string) string {
 	return base64.StdEncoding.EncodeToString([]byte(input))
 }
+
+type flowHideMatcher struct{}
+
+func (t *flowHideMatcher) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	// this matcher should not return any results
+	return expectResultsCount(results, 0)
+}

cmd/integration-test/javascript.go

@@ -14,6 +14,7 @@ var jsTestcases = []TestCaseInfo{
 	{Path: "protocols/javascript/redis-pass-brute.yaml", TestCase: &javascriptRedisPassBrute{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/ssh-server-fingerprint.yaml", TestCase: &javascriptSSHServerFingerprint{}, DisableOn: func() bool { return osutils.IsWindows() || osutils.IsOSX() }},
 	{Path: "protocols/javascript/net-multi-step.yaml", TestCase: &networkMultiStep{}},
+	{Path: "protocols/javascript/net-https.yaml", TestCase: &javascriptNetHttps{}},
 }
 
 var (
@@ -23,6 +24,16 @@ var (
 	defaultRetry  = 3
 )
 
+type javascriptNetHttps struct{}
+
+func (j *javascriptNetHttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}
+
 type javascriptRedisPassBrute struct{}
 
 func (j *javascriptRedisPassBrute) Execute(filePath string) error {

cmd/integration-test/network.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"fmt"
 	"net"
+	"os"
 	"strings"
+	"time"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
 	osutils "github.com/projectdiscovery/utils/os"
+	"github.com/projectdiscovery/utils/reader"
 )
 
 var networkTestcases = []TestCaseInfo{
@@ -16,6 +20,8 @@ var networkTestcases = []TestCaseInfo{
 	{Path: "protocols/network/variables.yaml", TestCase: &networkVariables{}},
 	{Path: "protocols/network/same-address.yaml", TestCase: &networkBasic{}},
 	{Path: "protocols/network/network-port.yaml", TestCase: &networkPort{}},
+	{Path: "protocols/network/net-https.yaml", TestCase: &networkhttps{}},
+	{Path: "protocols/network/net-https-timeout.yaml", TestCase: &networkhttps{}},
 }
 
 const defaultStaticPort = 5431
@@ -29,22 +35,26 @@ func (h *networkBasic) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			routerErr = err
 			return
 		}
 		if string(data) == "PING" {
 			_, _ = conn.Write([]byte("PONG"))
+		} else {
+			routerErr = fmt.Errorf("invalid data received: %s", string(data))
 		}
 	})
 	defer ts.Close()
 
 	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not run nuclei: %s\n", err)
 		return err
 	}
 	if routerErr != nil {
+		fmt.Fprintf(os.Stderr, "routerErr: %s\n", routerErr)
 		return routerErr
 	}
 
@@ -60,8 +70,8 @@ func (h *networkMultiStep) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 5)
+		data, err := reader.ConnReadNWithTimeout(conn, 5, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -69,8 +79,8 @@ func (h *networkMultiStep) Execute(filePath string) error {
 			_, _ = conn.Write([]byte("PING"))
 		}
 
-		data = make([]byte, 6)
+		data, err = reader.ConnReadNWithTimeout(conn, 6, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -126,8 +136,8 @@ func (h *networkVariables) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			routerErr = err
 			return
 		}
@@ -154,8 +164,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts := testutils.NewTCPServer(nil, 23846, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -187,8 +197,8 @@ func (n *networkPort) Execute(filePath string) error {
 	ts2 := testutils.NewTCPServer(nil, 34567, func(conn net.Conn) {
 		defer conn.Close()
 
-		data := make([]byte, 4)
+		data, err := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
-		if _, err := conn.Read(data); err != nil {
+		if err != nil {
 			return
 		}
 		if string(data) == "PING" {
@@ -206,3 +216,14 @@ func (n *networkPort) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type networkhttps struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *networkhttps) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(results, 1)
+}

go.mod

@@ -91,7 +91,7 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/utils v0.0.58
+	github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6
 	github.com/projectdiscovery/wappalyzergo v0.0.109
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/ropnop/gokrb5/v8 v8.0.0-20201111231119-729746023c02
@@ -166,7 +166,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
-	github.com/minio/selfupdate v0.6.0 // indirect
+	github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect

go.sum

@@ -679,8 +679,8 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
 github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
-github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
+github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
-github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
+github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -722,8 +722,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
 github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -837,8 +837,8 @@ github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766 h1:wa2wak7
 github.com/projectdiscovery/tlsx v1.1.6-0.20231016194953-a3ff9518c766/go.mod h1:bFATagikCvdPOsmaN1h5VQSbZjTW8bCQ6bjoQEePUq8=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/utils v0.0.58 h1:kk2AkSO84QZc9rDRI8jWA2Iia4uzb4sUcfh4h0xA20I=
+github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6 h1:60DKG3aueYiy93ZPt78yyZW0N+b7pWbK8Ub1UH6o08I=
-github.com/projectdiscovery/utils v0.0.58/go.mod h1:rsR5Kzjrb+/Yp7JSnEblLk4LfU4zH5Z7wQn8RzaGSdY=
+github.com/projectdiscovery/utils v0.0.61-0.20231031205429-0bc6a3c60ca6/go.mod h1:vt4oY4rvRWTdkBMhLlAGPbapa/R8pa+xZBYuNZIKJgQ=
 github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k=
 github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=

integration_tests/flow/flow-hide-matcher.yaml

@@ -0,0 +1,28 @@
+id: flow-hide-matcher
+
+info:
+  name: Test HTTP Template
+  author: pdteam
+  severity: info
+  description: In flow matcher output of previous step is hidden and only last event matcher output is shown
+
+flow: http(1) && http(2)
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - ok
+
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+
+    matchers:
+      - type: word
+        words:
+          - "Failed event"

integration_tests/protocols/code/py-env-var.yaml

@@ -20,4 +20,4 @@ code:
       - type: word
         words:
           - "hello from input baz"
-# digest: 4a0a00473045022100d407a3b848664b4c271abb4462a89a53fa2da6c21fd66011974ac395e2dc041c0220129a752a792337f6efe2e96562989016fe2709820b9583fd933f02be3b9d074f:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-file.yaml

@@ -18,4 +18,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221009db4541aa2af10aae5f39fe6e8789e2717c96ebbdadfdf33114ec0e82ec4da73022100fa98ee6611b606befc139946a169cca717f16ebf71beac97fdde1fe0c7fba774:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-interactsh.yaml

@@ -26,4 +26,4 @@ code:
         part: interactsh_protocol
         words:
           - "http"
-# digest: 4a0a0047304502205ebee72972ea0005ecdbcf7cd676ab861f3a44477a4b85dc1e745b7a628d2d7a022100ec4604673a1d43311ab343005464be5d4ee26b5a1f39206aa841056f3e2057dd:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/code/py-snippet.yaml

@@ -21,4 +21,4 @@ code:
       - type: word
         words:
           - "hello from input"
-# digest: 4b0a004830460221009a87b77e770e688bb1ce05e75ac075cdb3f318aad18a6dbc3fc2ec729a8ba5990221009020d69ba3baf47f9d835d4b6bd644a9e4f2d699369acc2a15983f5c270d2e79:4a3eb6b4988d95847d4203be25ed1d46
+# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46

integration_tests/protocols/javascript/net-https.yaml

@@ -0,0 +1,25 @@
+id: net-https
+
+info:
+  name: net-https
+  author: pdteam
+  severity: info
+  description: send and receive https data using net module
+
+
+javascript:
+  - code: |
+      let m = require('nuclei/net');
+      let name=Host+':'+Port;
+      let conn = m.OpenTLS('tcp', name);
+      conn.Send('GET / HTTP/1.1\r\nHost:'+name+'\r\nConnection: close\r\n\r\n');
+      resp = conn.RecvString();
+
+    args:
+      Host: "{{Host}}"
+      Port: "443"
+
+    matchers:
+      - type: word
+        words:
+          - "HTTP/1.1 200 OK"

integration_tests/protocols/network/net-https-timeout.yaml

@@ -0,0 +1,25 @@
+id: net-https-timeout
+
+info:
+  name: Example Network template which times out
+  author: pdteam
+  severity: high
+  description: Example Network template to send HTTPS request which times out
+
+
+tcp:
+  - host: 
+      - "tls://{{Hostname}}"
+    port: 443
+    inputs:
+      # noticable difference between this and net-https.yaml is that here we don't send the Connection: close header
+      # and hence connection will remain open until server closes it. This can be a DOS vector in nuclei
+      # as it waits for server to close the connection. now we have set a default timeout of 5 seconds and if server responds but doesn't close the connection
+      # then nuclei will close connection but doesn't fail the request since we already have response data from server
+      # this feature is only required for `read-all: true` to work properly
+      - data: "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n"
+    read-all: true
+    extractors:
+    - type: dsl
+      dsl:
+        - "len(data)"

integration_tests/protocols/network/net-https.yaml

@@ -0,0 +1,20 @@
+id: net-https
+
+info:
+  name: Example Network template to send HTTPS request
+  author: pdteam
+  severity: high
+  description: Example Network template to send HTTPS request
+
+
+tcp:
+  - host: 
+      - "tls://{{Hostname}}"
+    port: 443
+    inputs:
+      - data: "GET / HTTP/1.1\r\nHost: {{Hostname}}\r\nConnection: close\r\n\r\n"
+    read-all: true
+    extractors:
+    - type: dsl
+      dsl:
+        - "len(data)"

internal/runner/runner.go

@@ -692,7 +692,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 			if k != templates.Unsigned {
 				gologger.Info().Msgf("Executing %d signed templates from %s", v.Load(), k)
 			} else if !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning {
-				gologger.DefaultLogger.Print().Msgf("[%v] Executing %d unsigned templates. Use with caution.", aurora.BrightYellow("WRN"), v.Load())
+				gologger.Print().Msgf("[%v] Executing %d unsigned templates. Use with caution.", aurora.BrightYellow("WRN"), v.Load())
 			}
 		}
 	}

lib/example_test.go

@@ -72,6 +72,7 @@ func ExampleThreadSafeNucleiEngine() {
 
 	// Output:
 	// [nameserver-fingerprint] scanme.sh
+	// [caa-fingerprint] honey.scanme.sh
 }
 
 func TestMain(m *testing.M) {

pkg/js/generated/js/libnet/net.js

@@ -20,8 +20,8 @@ class NetConn {
 
     /**
     * @method
-    * @description Recv receives data from the connection with a timeout. If N is 0, it will read up to 4096 bytes.
+    * @description Recv receives data from the connection with a timeout. If N is 0, it will read all available data.
-    * @param {number} N - The number of bytes to receive.
+    * @param {number} [N=0] - The number of bytes to receive.
     * @returns {Uint8Array} - The received data in an array.
     * @throws {error} - The error encountered during data receiving.
     * @example
@@ -35,8 +35,8 @@ class NetConn {
 
     /**
     * @method
-    * @description RecvHex receives data from the connection with a timeout in hex format. If N is 0, it will read up to 4096 bytes.
+    * @description RecvHex receives data from the connection with a timeout in hex format. If N is 0, it will read all available data.
-    * @param {number} N - The number of bytes to receive.
+    * @param {number} [N=0] - The number of bytes to receive.
     * @returns {string} - The received data in hex format.
     * @throws {error} - The error encountered during data receiving.
     * @example
@@ -50,8 +50,8 @@ class NetConn {
 
     /**
     * @method
-    * @description RecvString receives data from the connection with a timeout. Output is returned as a string. If N is 0, it will read up to 4096 bytes.
+    * @description RecvString receives data from the connection with a timeout. Output is returned as a string. If N is 0, it will read all available data.
-    * @param {number} N - The number of bytes to receive.
+    * @param {number} [N=0] - The number of bytes to receive.
     * @returns {string} - The received data as a string.
     * @throws {error} - The error encountered during data receiving.
     * @example

pkg/js/libs/net/net.go

@@ -4,14 +4,18 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"net"
-	"syscall"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	errorutil "github.com/projectdiscovery/utils/errors"
+	"github.com/projectdiscovery/utils/reader"
+)
+
+var (
+	defaultTimeout = time.Duration(5) * time.Second
 )
 
 // Open opens a new connection to the address with a timeout.
@@ -21,13 +25,13 @@ func Open(protocol, address string) (*NetConn, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &NetConn{conn: conn}, nil
+	return &NetConn{conn: conn, timeout: defaultTimeout}, nil
 }
 
 // Open opens a new connection to the address with a timeout.
 // supported protocols: tcp, udp
 func OpenTLS(protocol, address string) (*NetConn, error) {
-	config := &tls.Config{InsecureSkipVerify: true}
+	config := &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}
 	host, _, _ := net.SplitHostPort(address)
 	if host != "" {
 		c := config.Clone()
@@ -38,7 +42,7 @@ func OpenTLS(protocol, address string) (*NetConn, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &NetConn{conn: conn}, nil
+	return &NetConn{conn: conn, timeout: defaultTimeout}, nil
 }
 
 // NetConn is a connection to a remote host.
@@ -67,9 +71,15 @@ func (c *NetConn) setDeadLine() {
 	_ = c.conn.SetDeadline(time.Now().Add(c.timeout))
 }
 
+// unsetDeadLine unsets read/write deadline for the connection.
+func (c *NetConn) unsetDeadLine() {
+	_ = c.conn.SetDeadline(time.Time{})
+}
+
 // SendArray sends array data to connection
 func (c *NetConn) SendArray(data []interface{}) error {
 	c.setDeadLine()
+	defer c.unsetDeadLine()
 	input := types.ToByteSlice(data)
 	length, err := c.conn.Write(input)
 	if err != nil {
@@ -84,6 +94,7 @@ func (c *NetConn) SendArray(data []interface{}) error {
 // SendHex sends hex data to connection
 func (c *NetConn) SendHex(data string) error {
 	c.setDeadLine()
+	defer c.unsetDeadLine()
 	bin, err := hex.DecodeString(data)
 	if err != nil {
 		return err
@@ -101,6 +112,7 @@ func (c *NetConn) SendHex(data string) error {
 // Send sends data to the connection with a timeout.
 func (c *NetConn) Send(data string) error {
 	c.setDeadLine()
+	defer c.unsetDeadLine()
 	bin := []byte(data)
 	length, err := c.conn.Write(bin)
 	if err != nil {
@@ -113,30 +125,24 @@ func (c *NetConn) Send(data string) error {
 }
 
 // Recv receives data from the connection with a timeout.
-// If N is 0, it will read up to 4096 bytes.
+// If N is 0, it will read all data sent by the server with 8MB limit.
 func (c *NetConn) Recv(N int) ([]byte, error) {
 	c.setDeadLine()
-	var response []byte
+	defer c.unsetDeadLine()
-	if N > 0 {
+	if N == 0 {
-		response = make([]byte, N)
+		// in utils we use -1 to indicate read all rather than 0
-	} else {
+		N = -1
-		response = make([]byte, 4096)
 	}
-	length, err := c.conn.Read(response)
+	bin, err := reader.ConnReadNWithTimeout(c.conn, int64(N), c.timeout)
 	if err != nil {
-		var netErr net.Error
+		return []byte{}, errorutil.NewWithErr(err).Msgf("failed to read %d bytes", N)
-		if (errors.As(err, &netErr) && netErr.Timeout()) ||
-			errors.Is(err, syscall.ECONNREFUSED) { // timeout error or connection refused
-			return response, nil
 	}
-		return response[:length], err
+	return bin, nil
-	}
-	return response[:length], nil
 }
 
 // RecvString receives data from the connection with a timeout
 // output is returned as a string.
-// If N is 0, it will read up to 4096 bytes.
+// If N is 0, it will read all data sent by the server with 8MB limit.
 func (c *NetConn) RecvString(N int) (string, error) {
 	bin, err := c.Recv(N)
 	if err != nil {
@@ -147,7 +153,7 @@ func (c *NetConn) RecvString(N int) (string, error) {
 
 // RecvHex receives data from the connection with a timeout
 // in hex format.
-// If N is 0, it will read up to 4096 bytes.
+// If N is 0,it will read all data sent by the server with 8MB limit.
 func (c *NetConn) RecvHex(N int) (string, error) {
 	bin, err := c.Recv(N)
 	if err != nil {

pkg/js/libs/smb/smbghost.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/structs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/utils/reader"
 )
 
 const (
@@ -31,10 +32,8 @@ func (c *SMBClient) DetectSMBGhost(host string, port int) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-
+	buff, _ := reader.ConnReadNWithTimeout(conn, 4, time.Duration(5)*time.Second)
-	buff := make([]byte, 4)
+	args, err := structs.Unpack(">I", buff)
-	nb, _ := conn.Read(buff)
-	args, err := structs.Unpack(">I", buff[:nb])
 	if err != nil {
 		return false, err
 	}
@@ -43,13 +42,14 @@ func (c *SMBClient) DetectSMBGhost(host string, port int) (bool, error) {
 	}
 
 	length := args[0].(int)
-	data := make([]byte, length)
 	_ = conn.SetReadDeadline(time.Now().Add(2 * time.Second))
-	n, err := conn.Read(data)
+	data, err := reader.ConnReadNWithTimeout(conn, int64(length), time.Duration(5)*time.Second)
 	if err != nil {
 		return false, err
 	}
-	data = data[:n]
+	if len(data) < 72 {
+		return false, errors.New("invalid response expected at least 72 bytes")
+	}
 
 	if !bytes.Equal(data[68:70], []byte("\x11\x03")) || !bytes.Equal(data[70:72], []byte("\x02\x00")) {
 		return false, nil

pkg/protocols/javascript/js.go

@@ -435,7 +435,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 	request.options.Progress.IncrementRequests()
 
 	requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
-	gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.TemplateID, hostPort)
+	gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort)
 
 	if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
 		msg := fmt.Sprintf("[%s] Dumped Javascript request for %s:\nVariables:\n %v", requestOptions.TemplateID, input.MetaInput.Input, vardump.DumpVariables(argsCopy.Args))

pkg/protocols/network/request.go

@@ -4,10 +4,8 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"io"
 	"net"
 	"net/url"
-	"os"
 	"strings"
 	"time"
 
@@ -30,6 +28,13 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	mapsutil "github.com/projectdiscovery/utils/maps"
+	"github.com/projectdiscovery/utils/reader"
+)
+
+var (
+	// TODO: make this configurable
+	// DefaultReadTimeout is the default read timeout for network requests
+	DefaultReadTimeout = time.Duration(5) * time.Second
 )
 
 var _ protocols.Request = &Request{}
@@ -196,15 +201,14 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		}
 
 		if input.Read > 0 {
-			buffer := make([]byte, input.Read)
+			buffer, err := reader.ConnReadNWithTimeout(conn, int64(input.Read), DefaultReadTimeout)
-			n, err := conn.Read(buffer)
 			if err != nil {
 				return errorutil.NewWithErr(err).Msgf("could not read response from connection")
 			}
 
-			responseBuilder.Write(buffer[:n])
+			responseBuilder.Write(buffer)
 
-			bufferStr := string(buffer[:n])
+			bufferStr := string(buffer)
 			if input.Name != "" {
 				inputEvents[input.Name] = bufferStr
 				interimValues[input.Name] = bufferStr
@@ -243,51 +247,19 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	if request.ReadSize != 0 {
 		bufferSize = request.ReadSize
 	}
-
-	var (
-		final []byte
-		n     int
-	)
-
 	if request.ReadAll {
-		readInterval := time.NewTimer(time.Second * 1)
+		bufferSize = -1
-		// stop the timer and drain the channel
-		closeTimer := func(t *time.Timer) {
-			if !t.Stop() {
-				<-t.C
 	}
-		}
+
-	readSocket:
+	final, err := reader.ConnReadNWithTimeout(conn, int64(bufferSize), DefaultReadTimeout)
-		for {
+	if err != nil {
-			select {
-			case <-readInterval.C:
-				closeTimer(readInterval)
-				break readSocket
-			default:
-				buf := make([]byte, bufferSize)
-				nBuf, err := conn.Read(buf)
-				if err != nil && !os.IsTimeout(err) && err != io.EOF {
-					request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
-					closeTimer(readInterval)
-					return errors.Wrap(err, "could not read from server")
-				}
-				responseBuilder.Write(buf[:nBuf])
-				final = append(final, buf...)
-				n += nBuf
-			}
-		}
-	} else {
-		final = make([]byte, bufferSize)
-		n, err = conn.Read(final)
-		if err != nil && !os.IsTimeout(err) && err != io.EOF {
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		return errors.Wrap(err, "could not read from server")
 	}
-		responseBuilder.Write(final[:n])
+	responseBuilder.Write(final)
-	}
 
 	response := responseBuilder.String()
-	outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input.MetaInput.Input, actualAddress)
+	outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final), response, input.MetaInput.Input, actualAddress)
 	// add response fields to template context and merge templatectx variables to output event
 	request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
 	outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll())

pkg/templates/types/types.go

@@ -58,7 +58,7 @@ var protocolMappings = map[ProtocolType]string{
 	WebsocketProtocol:  "websocket",
 	WHOISProtocol:      "whois",
 	CodeProtocol:       "code",
-	JavascriptProtocol: "js",
+	JavascriptProtocol: "javascript",
 }
 
 func GetSupportedProtocolTypes() ProtocolTypes {

pkg/tmplexec/flow/flow_executor.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 
 	"github.com/dop251/goja"
@@ -29,6 +28,13 @@ var (
 	ErrInvalidRequestID = errorutil.NewWithFmt("[%s] invalid request id '%s' provided")
 )
 
+// ProtoOptions are options that can be passed to flow protocol callback
+// ex: dns(protoOptions) <- protoOptions are optional and can be anything
+type ProtoOptions struct {
+	protoName string
+	reqIDS    []string
+}
+
 // FlowExecutor is a flow executor for executing a flow
 type FlowExecutor struct {
 	input   *contextargs.Context
@@ -37,14 +43,13 @@ type FlowExecutor struct {
 	// javascript runtime reference and compiled program
 	jsVM      *goja.Runtime
 	program   *goja.Program                // compiled js program
+	lastEvent *output.InternalWrappedEvent // contains last event that was emitted
 
 	// protocol requests and their callback functions
 	allProtocols   map[string][]protocols.Request
 	protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js
-	callback       func(event *output.InternalWrappedEvent)           // result event callback
 
 	// logic related variables
-	wg      sync.WaitGroup
 	results *atomic.Bool
 	allErrs mapsutil.SyncLockMap[string, error]
 }
@@ -72,6 +77,8 @@ func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, o
 			allprotos[templateTypes.WHOISProtocol.String()] = append(allprotos[templateTypes.WHOISProtocol.String()], req)
 		case templateTypes.CodeProtocol:
 			allprotos[templateTypes.CodeProtocol.String()] = append(allprotos[templateTypes.CodeProtocol.String()], req)
+		case templateTypes.JavascriptProtocol:
+			allprotos[templateTypes.JavascriptProtocol.String()] = append(allprotos[templateTypes.JavascriptProtocol.String()], req)
 		default:
 			gologger.Error().Msgf("invalid request type %s", req.Type().String())
 		}
@@ -143,22 +150,10 @@ func (f *FlowExecutor) Compile() error {
 			}
 			for _, v := range call.Arguments {
 				switch value := v.Export().(type) {
-				case map[string]interface{}:
-					opts.LoadOptions(value)
 				default:
 					opts.reqIDS = append(opts.reqIDS, types.ToString(value))
 				}
 			}
-			// parallel execution of protocols
-			if opts.Async {
-				f.wg.Add(1)
-				go func() {
-					defer f.wg.Done()
-					f.requestExecutor(reqMap, opts)
-				}()
-				return f.jsVM.ToValue(true)
-			}
-
 			return f.jsVM.ToValue(f.requestExecutor(reqMap, opts))
 		}
 	}
@@ -174,7 +169,6 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p
 		}
 	}()
 
-	f.callback = callback
 	f.input = input
 	// -----Load all types of variables-----
 	// add all input args to template context
@@ -183,7 +177,7 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p
 			f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value)
 		})
 	}
-	if f.callback == nil {
+	if callback == nil {
 		return fmt.Errorf("output callback cannot be nil")
 	}
 	// pass flow and execute the js vm and handle errors
@@ -191,11 +185,12 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow)
 	}
-	f.wg.Wait()
 	runtimeErr := f.GetRuntimeErrors()
 	if runtimeErr != nil {
 		return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow")
 	}
+	// this is where final result is generated/created
+	callback(f.lastEvent)
 	if value.Export() != nil {
 		f.results.Store(value.ToBoolean())
 	} else {

pkg/tmplexec/flow/flow_internal.go

@@ -34,34 +34,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req
 		// execution logic for http()/dns() etc
 		for index := range f.allProtocols[opts.protoName] {
 			req := f.allProtocols[opts.protoName][index]
-			err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) {
+			err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts))
-				if result != nil {
-					f.results.CompareAndSwap(false, true)
-					if !opts.Hide {
-						f.callback(result)
-					}
-					// export dynamic values from operators (i.e internal:true)
-					// add add it to template context
-					// this is a conflicting behaviour with iterate-all
-					if result.HasOperatorResult() {
-						matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched)
-						if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) {
-							// if matcher status is false . check if template/request contains any matcher at all
-							// if it does then we need to set matcher status to true
-							matcherStatus.CompareAndSwap(false, true)
-						}
-						if len(result.OperatorsResult.DynamicValues) > 0 {
-							for k, v := range result.OperatorsResult.DynamicValues {
-								f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v)
-							}
-						}
-					} else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) {
-						// if matcher status is false . check if template/request contains any matcher at all
-						// if it does then we need to set matcher status to true
-						matcherStatus.CompareAndSwap(false, true)
-					}
-				}
-			})
 			if err != nil {
 				// save all errors in a map with id as key
 				// its less likely that there will be race condition but just in case
@@ -90,25 +63,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req
 			}
 			return matcherStatus.Load()
 		}
-		err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) {
+		err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, f.getProtoRequestCallback(req, matcherStatus, opts))
-			if result != nil {
-				f.results.CompareAndSwap(false, true)
-				if !opts.Hide {
-					f.callback(result)
-				}
-				// export dynamic values from operators (i.e internal:true)
-				// add add it to template context
-				if result.HasOperatorResult() {
-					matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched)
-					if len(result.OperatorsResult.DynamicValues) > 0 {
-						for k, v := range result.OperatorsResult.DynamicValues {
-							f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v)
-						}
-						_ = f.jsVM.Set("template", f.options.GetTemplateCtx(f.input.MetaInput).GetAll())
-					}
-				}
-			}
-		})
 		if err != nil {
 			index := id
 			err = f.allErrs.Set(opts.protoName+":"+index, err)
@@ -120,6 +75,39 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req
 	return matcherStatus.Load()
 }
 
+// getProtoRequestCallback returns a callback that is executed
+// after execution of each protocol request
+func (f *FlowExecutor) getProtoRequestCallback(req protocols.Request, matcherStatus *atomic.Bool, opts *ProtoOptions) func(result *output.InternalWrappedEvent) {
+	return func(result *output.InternalWrappedEvent) {
+		if result != nil {
+			f.results.CompareAndSwap(false, true)
+			f.lastEvent = result
+			// export dynamic values from operators (i.e internal:true)
+			// add add it to template context
+			// this is a conflicting behaviour with iterate-all
+			if result.HasOperatorResult() {
+				// this is to handle case where there is any operator result (matcher or extractor)
+				matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched)
+				if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) {
+					// if matcher status is false . check if template/request contains any matcher at all
+					// if it does then we need to set matcher status to true
+					matcherStatus.CompareAndSwap(false, true)
+				}
+				if len(result.OperatorsResult.DynamicValues) > 0 {
+					for k, v := range result.OperatorsResult.DynamicValues {
+						f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v)
+					}
+				}
+			} else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) {
+				// this is to handle case where there are no operator result and there was no matcher in operators
+				// if matcher status is false . check if template/request contains any matcher at all
+				// if it does then we need to set matcher status to true
+				matcherStatus.CompareAndSwap(false, true)
+			}
+		}
+	}
+}
+
 // registerBuiltInFunctions registers all built in functions for the flow
 func (f *FlowExecutor) registerBuiltInFunctions() error {
 	// currently we register following builtin functions

pkg/tmplexec/flow/options.go

@@ -1,48 +0,0 @@
-package flow
-
-import (
-	"strings"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// ProtoOptions are options that can be passed to flow protocol callback
-// ex: dns(protoOptions) <- protoOptions are optional and can be anything
-type ProtoOptions struct {
-	Hide      bool
-	Async     bool
-	protoName string
-	reqIDS    []string
-}
-
-// Examples
-//  dns() <- callback without any options
-//  dns(1) or dns(1,3) <- callback with index of protocol in template request at 1 or  1 and 3
-//  dns("probe-http") or dns("extract-vpc","probe-http") <- callback with id's instead of index of request in template
-//  dns({hide:true}) or dns({hide:true,async:true}) <- callback with protocol options
-//  hide - hides result/event from output & sdk
-//  async - executes protocols in parallel (implicit wait no need to specify wait)
-// Note: all of these options are optional and can be combined together in any order
-
-// LoadOptions loads the protocol options from a map
-func (P *ProtoOptions) LoadOptions(m map[string]interface{}) {
-	P.Hide = GetBool(m["hide"])
-	P.Async = GetBool(m["async"])
-}
-
-// GetBool returns bool value from interface
-func GetBool(value interface{}) bool {
-	if value == nil {
-		return false
-	}
-	switch v := value.(type) {
-	case bool:
-		return v
-	default:
-		tmpValue := types.ToString(value)
-		if strings.EqualFold(tmpValue, "true") {
-			return true
-		}
-	}
-	return false
-}