From deaa86f5cc3beb500d7bfcd2aad8078c07bac4ec Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Thu, 16 Sep 2021 21:29:58 +0530
Subject: [PATCH 01/23] Added interactsh based template to integration test

---
 integration_tests/http/interactsh.yaml | 19 +++++++++++++++++
 v2/cmd/integration-test/http.go        | 29 ++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 integration_tests/http/interactsh.yaml

diff --git a/integration_tests/http/interactsh.yaml b/integration_tests/http/interactsh.yaml
new file mode 100644
index 000000000..28d9c5606
--- /dev/null
+++ b/integration_tests/http/interactsh.yaml
@@ -0,0 +1,19 @@
+id: interactsh-integration-test
+
+info:
+  name: Interactsh Integration Test
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      url: 'http://{{interactsh-url}}'
+
+    matchers:
+      - type: word
+        part: interactsh_protocol # Confirms the HTTP Interaction
+        words:
+          - "http"
\ No newline at end of file
diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
index 41f177d28..37f1fadf0 100644
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@@ -32,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
 	"http/request-condition.yaml":     &httpRequestCondition{},
 	"http/request-condition-new.yaml": &httpRequestCondition{},
+	"http/interactsh.yaml":            &httpInteractshRequest{},
 }
 
 func httpDebugRequestDump(r *http.Request) {
@@ -42,6 +43,34 @@ func httpDebugRequestDump(r *http.Request) {
 	}
 }
 
+type httpInteractshRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpInteractshRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		httpDebugRequestDump(r)
+
+		value := r.Header.Get("url")
+		if value != "" {
+			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+				resp.Body.Close()
+			}
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
 type httpGetHeaders struct{}
 
 // Executes executes a test case and returns an error if occurred

From f8d5d025702da2e9a477b0f04b2602cd2053abc2 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 12 Oct 2021 23:28:24 +0200
Subject: [PATCH 02/23] Adding support to read tcp data stream till the end

---
 v2/pkg/protocols/network/network.go |  7 +++++
 v2/pkg/protocols/network/request.go | 40 ++++++++++++++++++++++++-----
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go
index b7de7b920..ef47dc36e 100644
--- a/v2/pkg/protocols/network/network.go
+++ b/v2/pkg/protocols/network/network.go
@@ -60,6 +60,13 @@ type Request struct {
 	// examples:
 	//   - value: "2048"
 	ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"`
+	// description: |
+	//   ReadAll determines if the data stream should be read till the end regardless of the size
+	//
+	//   Default value for read-all is false.
+	// examples:
+	//   - value: false
+	ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"`
 
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go
index aad3c60ef..ab5176685 100644
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"net"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -183,13 +184,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
 	if r.ReadSize != 0 {
 		bufferSize = r.ReadSize
 	}
-	final := make([]byte, bufferSize)
-	n, err := conn.Read(final)
-	if err != nil && err != io.EOF {
-		r.options.Output.Request(r.options.TemplateID, address, "network", err)
-		return errors.Wrap(err, "could not read from server")
+
+	var (
+		final []byte
+		n     int
+	)
+
+	if r.ReadAll {
+		readInterval := time.After(time.Second * 1)
+	read_socket:
+		for {
+			select {
+			case <-readInterval:
+				break read_socket
+			default:
+				buf := make([]byte, bufferSize)
+				nBuf, err := conn.Read(buf)
+				if err != nil && !os.IsTimeout(err) {
+					r.options.Output.Request(r.options.TemplateID, address, "network", err)
+					return errors.Wrap(err, "could not read from server")
+				}
+				responseBuilder.Write(buf[:nBuf])
+				final = append(final, buf...)
+				n += nBuf
+			}
+		}
+	} else {
+		final = make([]byte, bufferSize)
+		n, err = conn.Read(final)
+		if err != nil && err != io.EOF {
+			r.options.Output.Request(r.options.TemplateID, address, "network", err)
+			return errors.Wrap(err, "could not read from server")
+		}
+		responseBuilder.Write(final[:n])
 	}
-	responseBuilder.Write(final[:n])
 
 	if r.options.Options.Debug || r.options.Options.DebugResponse {
 		responseOutput := responseBuilder.String()

From 5584fc285d6377bc0fb53bf14385a1f7b09bef10 Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Tue, 19 Oct 2021 00:58:49 +0300
Subject: [PATCH 03/23] Add regexp pattern for template-id

---
 v2/pkg/templates/templates.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go
index b6f2491ac..0b925d7e7 100644
--- a/v2/pkg/templates/templates.go
+++ b/v2/pkg/templates/templates.go
@@ -27,7 +27,7 @@ type Template struct {
 	// examples:
 	//   - name: ID Example
 	//     value: "\"CVE-2021-19520\""
-	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"`
+	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"`
 	// description: |
 	//   Info contains metadata information about the template.
 	// examples:

From 0ba3b19f1f7653a41327315644b7fcccb57de1c2 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 19 Oct 2021 11:31:36 +0200
Subject: [PATCH 04/23] fixing merge issues

---
 v2/pkg/protocols/network/request.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go
index 399699a0f..a1425ebae 100644
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@@ -191,13 +191,13 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	if request.ReadSize != 0 {
 		bufferSize = request.ReadSize
 	}
-  
-  var (
+
+	var (
 		final []byte
 		n     int
 	)
 
-	if r.ReadAll {
+	if request.ReadAll {
 		readInterval := time.After(time.Second * 1)
 	read_socket:
 		for {
@@ -208,7 +208,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 				buf := make([]byte, bufferSize)
 				nBuf, err := conn.Read(buf)
 				if err != nil && !os.IsTimeout(err) {
-					r.options.Output.Request(r.options.TemplateID, address, "network", err)
+					request.options.Output.Request(request.options.TemplateID, address, "network", err)
 					return errors.Wrap(err, "could not read from server")
 				}
 				responseBuilder.Write(buf[:nBuf])

From ae99a88f52b4ddb260818a8ba80a28f650233894 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Tue, 19 Oct 2021 19:39:33 +0200
Subject: [PATCH 05/23] timer.after => timer

---
 v2/pkg/protocols/network/request.go | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go
index a1425ebae..6beecc6e1 100644
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@@ -198,17 +198,25 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	)
 
 	if request.ReadAll {
-		readInterval := time.After(time.Second * 1)
+		readInterval := time.NewTimer(time.Second * 1)
+		// stop the timer and drain the channel
+		closeTimer := func(t *time.Timer) {
+			if !t.Stop() {
+				<-t.C
+			}
+		}
 	read_socket:
 		for {
 			select {
-			case <-readInterval:
+			case <-readInterval.C:
+				closeTimer(readInterval)
 				break read_socket
 			default:
 				buf := make([]byte, bufferSize)
 				nBuf, err := conn.Read(buf)
 				if err != nil && !os.IsTimeout(err) {
 					request.options.Output.Request(request.options.TemplateID, address, "network", err)
+					closeTimer(readInterval)
 					return errors.Wrap(err, "could not read from server")
 				}
 				responseBuilder.Write(buf[:nBuf])

From a1e099493dc3448465eccb7ee3072487fd03403f Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Wed, 20 Oct 2021 00:43:33 +0300
Subject: [PATCH 06/23] Use simpler regexp for template id validation

---
 v2/pkg/templates/templates.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go
index 0b925d7e7..a9a0e14ea 100644
--- a/v2/pkg/templates/templates.go
+++ b/v2/pkg/templates/templates.go
@@ -27,7 +27,7 @@ type Template struct {
 	// examples:
 	//   - name: ID Example
 	//     value: "\"CVE-2021-19520\""
-	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"`
+	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"`
 	// description: |
 	//   Info contains metadata information about the template.
 	// examples:

From cbc5192e9a3e1d7f5409385ef26885d0154c1c6e Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Wed, 20 Oct 2021 00:02:06 +0200
Subject: [PATCH 07/23] Adding proxy socks support to headless browser

---
 .../protocols/headless/engine/http_client.go  | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go
index b4747d541..9da2aaa93 100644
--- a/v2/pkg/protocols/headless/engine/http_client.go
+++ b/v2/pkg/protocols/headless/engine/http_client.go
@@ -1,13 +1,17 @@
 package engine
 
 import (
+	"context"
 	"crypto/tls"
+	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"golang.org/x/net/proxy"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
@@ -28,6 +32,22 @@ func newhttpClient(options *types.Options) *http.Client {
 		if proxyURL, err := url.Parse(options.ProxyURL); err == nil {
 			transport.Proxy = http.ProxyURL(proxyURL)
 		}
+	} else if options.ProxySocksURL != "" {
+		var proxyAuth *proxy.Auth
+
+		socksURL, proxyErr := url.Parse(options.ProxySocksURL)
+		if proxyErr == nil {
+			proxyAuth = &proxy.Auth{}
+			proxyAuth.User = socksURL.User.Username()
+			proxyAuth.Password, _ = socksURL.User.Password()
+		}
+		dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
+		dc := dialer.(interface {
+			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+		})
+		if proxyErr == nil {
+			transport.DialContext = dc.DialContext
+		}
 	}
 
 	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}

From 9e0144b6aede383452969ae81a771d231dfdd6e2 Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Wed, 20 Oct 2021 01:31:38 +0300
Subject: [PATCH 08/23] Add template-id validation

---
 v2/pkg/parsers/parser.go      | 30 ++++++++++------
 v2/pkg/parsers/parser_test.go | 68 +++++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+), 11 deletions(-)
 create mode 100644 v2/pkg/parsers/parser_test.go

diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go
index b65cb79e2..1d5213ae3 100644
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@@ -17,7 +17,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 )
 
-const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+const (
+	mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+	invalidFieldFormatTemplate    = "invalid field format for '%s' (allowed format is %s)"
+)
 
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
 func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
@@ -30,12 +33,11 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 		return false, nil
 	}
 
-	templateInfo := template.Info
-	if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+	if validationError := validateTemplateFields(template); validationError != nil {
 		return false, validationError
 	}
 
-	return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
+	return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags)
 }
 
 // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@@ -45,10 +47,8 @@ func LoadWorkflow(templatePath string) (bool, error) {
 		return false, templateParseError
 	}
 
-	templateInfo := template.Info
-
 	if len(template.Workflows) > 0 {
-		if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		if validationError := validateTemplateFields(template); validationError != nil {
 			return false, validationError
 		}
 		return true, nil
@@ -71,10 +71,8 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode
 	return match, err
 }
 
-func validateMandatoryInfoFields(info *model.Info) error {
-	if info == nil {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
-	}
+func validateTemplateFields(template *templates.Template) error {
+	info := template.Info
 
 	if utils.IsBlank(info.Name) {
 		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
@@ -83,6 +81,15 @@ func validateMandatoryInfoFields(info *model.Info) error {
 	if info.Authors.IsEmpty() {
 		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
 	}
+
+	if template.ID == "" {
+		return fmt.Errorf(mandatoryFieldMissingTemplate, "id")
+	}
+
+	if !templateIDRegexp.MatchString(template.ID) {
+		return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())
+	}
+
 	return nil
 }
 
@@ -90,6 +97,7 @@ var (
 	parsedTemplatesCache *cache.Templates
 	ShouldValidate       bool
 	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
+	templateIDRegexp     = regexp.MustCompile(`^[A-Za-z0-9-_]+$`)
 )
 
 const (
diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go
new file mode 100644
index 000000000..e1e2a1282
--- /dev/null
+++ b/v2/pkg/parsers/parser_test.go
@@ -0,0 +1,68 @@
+package parsers
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplate(t *testing.T) {
+	origTemplatesCache := parsedTemplatesCache
+	defer func() { parsedTemplatesCache = origTemplatesCache }()
+
+	tt := []struct {
+		name        string
+		template    *templates.Template
+		templateErr error
+
+		expectedErr error
+	}{
+		{
+			name: "valid",
+			template: &templates.Template{
+				ID: "CVE-2021-27330",
+				Info: model.Info{
+					Name:    "Valid template",
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+		},
+		{
+			name:        "missingName",
+			template:    &templates.Template{},
+			expectedErr: errors.New("mandatory 'name' field is missing"),
+		},
+		{
+			name: "invalidID",
+			template: &templates.Template{
+				ID: "ABC DEF",
+				Info: model.Info{
+					Name:    "Invalid ID",
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+			expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr)
+
+			tagFilter := filter.New(&filter.Config{})
+			success, err := LoadTemplate(tc.name, tagFilter, nil)
+			if tc.expectedErr == nil {
+				require.NoError(t, err)
+				require.True(t, success)
+			} else {
+				require.Equal(t, tc.expectedErr, err)
+				require.False(t, success)
+			}
+		})
+	}
+}

From e6728e8ff9f4d1d25d91b0f9d047dbca84010308 Mon Sep 17 00:00:00 2001
From: mzack <marco.rivoli.nvh@gmail.com>
Date: Wed, 20 Oct 2021 13:26:47 +0200
Subject: [PATCH 09/23] Making headless httpclient more similar to real
 browsers

---
 v2/pkg/protocols/headless/engine/http_client.go | 15 ++++++++++++++-
 v2/pkg/protocols/headless/engine/rules.go       |  3 +++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go
index b4747d541..94c33563c 100644
--- a/v2/pkg/protocols/headless/engine/http_client.go
+++ b/v2/pkg/protocols/headless/engine/http_client.go
@@ -3,6 +3,7 @@ package engine
 import (
 	"crypto/tls"
 	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 	"time"
 
@@ -30,5 +31,17 @@ func newhttpClient(options *types.Options) *http.Client {
 		}
 	}
 
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
+	jar, _ := cookiejar.New(nil)
+
+	httpclient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Duration(options.Timeout*3) * time.Second,
+		Jar:       jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// the browser should follow redirects not us
+			return http.ErrUseLastResponse
+		},
+	}
+
+	return httpclient
 }
diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go
index a254902eb..a802a64a9 100644
--- a/v2/pkg/protocols/headless/engine/rules.go
+++ b/v2/pkg/protocols/headless/engine/rules.go
@@ -8,6 +8,9 @@ import (
 
 // routingRuleHandler handles proxy rule for actions related to request/response modification
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
+	// usually browsers don't use chunked transfer encoding so we set the content-length nevertheless
+	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
+
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue

From f1cd0a5d288de0fc4212287ec1f0a06033e9fc7b Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Wed, 20 Oct 2021 22:58:36 +0300
Subject: [PATCH 10/23] Update template id regexp

---
 v2/pkg/parsers/parser.go      | 2 +-
 v2/pkg/parsers/parser_test.go | 2 +-
 v2/pkg/templates/templates.go | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go
index 1d5213ae3..7bde724f0 100644
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@@ -97,7 +97,7 @@ var (
 	parsedTemplatesCache *cache.Templates
 	ShouldValidate       bool
 	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
-	templateIDRegexp     = regexp.MustCompile(`^[A-Za-z0-9-_]+$`)
+	templateIDRegexp     = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`)
 )
 
 const (
diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go
index e1e2a1282..fefde3237 100644
--- a/v2/pkg/parsers/parser_test.go
+++ b/v2/pkg/parsers/parser_test.go
@@ -46,7 +46,7 @@ func TestLoadTemplate(t *testing.T) {
 					Authors: stringslice.StringSlice{Value: "Author"},
 				},
 			},
-			expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"),
+			expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"),
 		},
 	}
 
diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go
index a9a0e14ea..8f5a4e167 100644
--- a/v2/pkg/templates/templates.go
+++ b/v2/pkg/templates/templates.go
@@ -27,7 +27,7 @@ type Template struct {
 	// examples:
 	//   - name: ID Example
 	//     value: "\"CVE-2021-19520\""
-	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"`
+	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
 	// description: |
 	//   Info contains metadata information about the template.
 	// examples:

From 5d0f6b2622f54aa8f1980f5ef83138523e23c26a Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Wed, 20 Oct 2021 23:14:04 +0300
Subject: [PATCH 11/23] Improve invalid template id tests

---
 v2/pkg/parsers/parser_test.go | 54 ++++++++++++++++++++++++++++-------
 1 file changed, 43 insertions(+), 11 deletions(-)

diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go
index fefde3237..e94b2ee30 100644
--- a/v2/pkg/parsers/parser_test.go
+++ b/v2/pkg/parsers/parser_test.go
@@ -2,6 +2,7 @@ package parsers
 
 import (
 	"errors"
+	"fmt"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
@@ -37,17 +38,6 @@ func TestLoadTemplate(t *testing.T) {
 			template:    &templates.Template{},
 			expectedErr: errors.New("mandatory 'name' field is missing"),
 		},
-		{
-			name: "invalidID",
-			template: &templates.Template{
-				ID: "ABC DEF",
-				Info: model.Info{
-					Name:    "Invalid ID",
-					Authors: stringslice.StringSlice{Value: "Author"},
-				},
-			},
-			expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"),
-		},
 	}
 
 	for _, tc := range tt {
@@ -65,4 +55,46 @@ func TestLoadTemplate(t *testing.T) {
 			}
 		})
 	}
+
+	t.Run("invalidTemplateID", func(t *testing.T) {
+		tt := []struct {
+			id      string
+			success bool
+		}{
+			{id: "A-B-C", success: true},
+			{id: "A-B-C-1", success: true},
+			{id: "CVE_2021_27330", success: true},
+			{id: "ABC DEF", success: false},
+			{id: "_-__AAA_", success: false},
+			{id: " CVE-2021-27330", success: false},
+			{id: "CVE-2021-27330 ", success: false},
+			{id: "CVE-2021-27330-", success: false},
+			{id: "-CVE-2021-27330-", success: false},
+			{id: "CVE-2021--27330", success: false},
+			{id: "CVE-2021+27330", success: false},
+		}
+		for i, tc := range tt {
+			name := fmt.Sprintf("regexp%d", i)
+			t.Run(name, func(t *testing.T) {
+				template := &templates.Template{
+					ID: tc.id,
+					Info: model.Info{
+						Name:    "Valid template",
+						Authors: stringslice.StringSlice{Value: "Author"},
+					},
+				}
+				parsedTemplatesCache.Store(name, template, nil)
+
+				tagFilter := filter.New(&filter.Config{})
+				success, err := LoadTemplate(name, tagFilter, nil)
+				if tc.success {
+					require.NoError(t, err)
+					require.True(t, success)
+				} else {
+					require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err)
+					require.False(t, success)
+				}
+			})
+		}
+	})
 }

From ff7a5997a225a81ed35b9017b76105ffe186111f Mon Sep 17 00:00:00 2001
From: Alexey Zhuchkov <zer0divisi1@gmail.com>
Date: Wed, 20 Oct 2021 23:24:11 +0300
Subject: [PATCH 12/23] Return multiple errors in template validation

---
 v2/pkg/parsers/parser.go      | 15 ++++++++++-----
 v2/pkg/parsers/parser_test.go | 14 ++++++++++++--
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go
index 7bde724f0..83059baa7 100644
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 
@@ -74,20 +75,24 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode
 func validateTemplateFields(template *templates.Template) error {
 	info := template.Info
 
+	var errors []string
+
 	if utils.IsBlank(info.Name) {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name"))
 	}
 
 	if info.Authors.IsEmpty() {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author"))
 	}
 
 	if template.ID == "" {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "id")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id"))
+	} else if !templateIDRegexp.MatchString(template.ID) {
+		errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()))
 	}
 
-	if !templateIDRegexp.MatchString(template.ID) {
-		return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ", "))
 	}
 
 	return nil
diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go
index e94b2ee30..ef74a317e 100644
--- a/v2/pkg/parsers/parser_test.go
+++ b/v2/pkg/parsers/parser_test.go
@@ -34,9 +34,19 @@ func TestLoadTemplate(t *testing.T) {
 			},
 		},
 		{
-			name:        "missingName",
+			name:        "emptyTemplate",
 			template:    &templates.Template{},
-			expectedErr: errors.New("mandatory 'name' field is missing"),
+			expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"),
+		},
+		{
+			name: "emptyNameWithInvalidID",
+			template: &templates.Template{
+				ID: "invalid id",
+				Info: model.Info{
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+			expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"),
 		},
 	}
 

From fa79a16a9d261b46f743b7773dc1d1304dcfba7d Mon Sep 17 00:00:00 2001
From: sandeep <sandeep@projectdiscovery.io>
Date: Thu, 21 Oct 2021 05:56:15 +0530
Subject: [PATCH 13/23] reverting #1017

---
 v2/internal/runner/options.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go
index 745f15d02..59791fe63 100644
--- a/v2/internal/runner/options.go
+++ b/v2/internal/runner/options.go
@@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
-	// if VerboseVerbose is set, it implicitly enables the Verbose option as well
-	if options.VerboseVerbose {
-		options.Verbose = true
-	}
-
 	// Read the inputs and configure the logging
 	configureOutput(options)
 
@@ -127,7 +122,7 @@ func isValidURL(urlString string) bool {
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose || options.VerboseVerbose {
+	if options.Verbose {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {

From dd311897fcbb61c46a6722de51a740f4599b56c6 Mon Sep 17 00:00:00 2001
From: sandeep <sandeep@projectdiscovery.io>
Date: Thu, 21 Oct 2021 06:05:09 +0530
Subject: [PATCH 14/23] misc flag update

---
 v2/cmd/nuclei/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go
index c5170f201..d2f8ebb9f 100644
--- a/v2/cmd/nuclei/main.go
+++ b/v2/cmd/nuclei/main.go
@@ -141,7 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
-		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
 		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
 	)
 

From f260ed6da78e2f8295c82d00c13b6c54f09c9970 Mon Sep 17 00:00:00 2001
From: sandeep <sandeep@projectdiscovery.io>
Date: Thu, 21 Oct 2021 06:17:18 +0530
Subject: [PATCH 15/23] readme + version update

---
 README.md                       | 75 +++++++++++++++++----------------
 v2/pkg/catalog/config/config.go |  2 +-
 2 files changed, 40 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 750e2a147..bfa68972e 100644
--- a/README.md
+++ b/README.md
@@ -90,30 +90,29 @@ TARGET:
    -l, -list string      path to file containing a list of target URLs/hosts to scan (one per line)
 
 TEMPLATES:
-   -tl                      list all available templates
    -t, -templates string[]  template or template directory paths to include in the scan
-   -w, -workflows string[]  list of workflows to run
-   -nt, -new-templates      run newly added templates only
+   -nt, -new-templates      run only new templates added in latest nuclei-templates release
+   -w, -workflows string[]  workflow or workflow directory paths to include in the scan
    -validate                validate the passed templates to nuclei
+   -tl                      list all available templates
 
 FILTERING:
-   -tags string[]                         execute a subset of templates that contain the provided tags
-   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
-   -etags, -exclude-tags string[]         exclude templates with the provided tags
-   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
-   -exclude-templates, -exclude string[]  template or template directory paths to exclude
-   -severity, -impact value[]             Templates to run based on severity. Possible values: info, low, medium, high, critical
-   -author string[]                       execute templates that are (co-)created by the specified authors
+   -tags string[]                    execute a subset of templates that contain the provided tags
+   -etags, -exclude-tags string[]    exclude templates with the provided tags
+   -itags, -include-tags string[]    tags from the default deny list that permit executing more intrusive templates
+   -et, -exclude-templates string[]  template or template directory paths to exclude
+   -it, -include-templates string[]  templates to be executed even if they are excluded either by default or configuration
+   -s, -severity value[]             Templates to run based on severity. Possible values - info,low,medium,high,critical
+   -es, -exclude-severity value[]    Templates to exclude based on severity. Possible values - info,low,medium,high,critical
+   -a, -author string[]              execute templates that are (co-)created by the specified authors
 
 OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -silent                       display findings only
-   -v, -verbose                  show verbose output
-   -vv                           display extra verbose information
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
-   -nm, -no-meta                 don't display match metadata in CLI output
+   -nm, -no-meta                 don't display match metadata
    -nts, -no-timestamp           don't display timestamp metadata in CLI output
    -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
    -me, -markdown-export string  directory to export results in markdown format
@@ -125,37 +124,39 @@ CONFIGURATIONS:
    -H, -header string[]        custom headers in header:value format
    -V, -var value              custom vars in var=value format
    -r, -resolvers string       file containing resolver list for nuclei
-   -system-resolvers           use system DNS resolving as error fallback
+   -sr, -system-resolvers      use system DNS resolving as error fallback
    -passive                    enable passive HTTP response processing mode
-   -env-vars                   enable environment variables support
+   -ev, -env-vars              enable environment variables to be used in template
 
 INTERACTSH:
-   -no-interactsh                     disable interactsh server for OOB testing
-   -interactsh-url string             interactsh server url for self-hosted instance (default "https://interactsh.com")
-   -interactsh-token string           authentication token for self-hosted interactsh server
-   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
-   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
-   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
-   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+   -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default "https://interactsh.com")
+   -itoken, -interactsh-token string    authentication token for self-hosted interactsh server
+   -interactions-cache-size int         number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int           number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int      number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int    extra time for interaction polling before exiting (default 5)
+   -ni, -no-interactsh                  disable interactsh server for OAST testing, exclude OAST based templates
 
 RATE-LIMIT:
    -rl, -rate-limit int          maximum number of requests to send per second (default 150)
    -rlm, -rate-limit-minute int  maximum number of requests to send per minute
    -bs, -bulk-size int           maximum number of hosts to be analyzed in parallel per template (default 25)
-   -c, -concurrency int          maximum number of templates to be executed in parallel (default 10)
+   -c, -concurrency int          maximum number of templates to be executed in parallel (default 25)
 
 OPTIMIZATIONS:
    -timeout int               time to wait in seconds before timeout (default 5)
    -retries int               number of times to retry a failed request (default 1)
-   -max-host-error int        max errors for a host before skipping from scan (default 30)
+   -mhe, -max-host-error int  max errors for a host before skipping from scan (default 30)
    -project                   use a project folder to avoid sending same request multiple times
-   -project-path string       set a specific project path (default "$TMPDIR/")
+   -project-path string       set a specific project path
    -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+   -stream                    Stream mode - start elaborating without sorting the input
 
 HEADLESS:
-   -headless          enable templates that require headless browser support
-   -page-timeout int  seconds to wait for each page in headless mode (default 20)
-   -show-browser      show the browser on the screen when running templates with headless mode
+   -headless            enable templates that require headless browser support
+   -page-timeout int    seconds to wait for each page in headless mode (default 20)
+   -sb, -show-browser   show the browser on the screen when running templates with headless mode
+   -sc, -system-chrome  Use local installed chrome browser instead of nuclei installed
 
 DEBUG:
    -debug                     show all requests and responses
@@ -163,22 +164,24 @@ DEBUG:
    -debug-resp                show all received responses
    -proxy, -proxy-url string  URL of the HTTP proxy server
    -proxy-socks-url string    URL of the SOCKS proxy server
-   -trace-log string          file to write sent requests trace log
+   -tlog, -trace-log string   file to write sent requests trace log
    -version                   show nuclei version
+   -v, -verbose               show verbose output
+   -vv                        display extra verbose information
    -tv, -templates-version    shows the version of the installed nuclei-templates
 
 UPDATE:
-   -update                        update nuclei to the latest released version
-   -ut, -update-templates         update the community templates to latest released version
-   -nut, -no-update-templates     do not check for nuclei-templates updates
-   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+   -update                        update nuclei engine to the latest released version
+   -ut, -update-templates         update nuclei-templates to latest released version
+   -ud, -update-directory string  overwrite the default directory to install nuclei-templates
+   -duc, -disable-update-check    disable automatic nuclei/templates update check
 
 STATISTICS:
    -stats                    display statistics about the running scan
-   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -sj, -stats-json          write statistics data to an output file in JSONL(ines) format
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
-   -metrics                  expose nuclei metrics on a port
-   -metrics-port int         port to expose nuclei metrics on (default 9092)
+   -m, -metrics              expose nuclei metrics on a port
+   -mp, -metrics-port int    port to expose nuclei metrics on (default 9092)
 ```
 
 ### Running Nuclei
diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go
index 41ce53613..8a996b62d 100644
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@@ -26,7 +26,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.5.3-dev`
+const Version = `2.5.3`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()

From fdb7c0d6cda8b2acf11af247d9eaf65f5b58f036 Mon Sep 17 00:00:00 2001
From: sandeep <sandeep@projectdiscovery.io>
Date: Thu, 21 Oct 2021 06:20:40 +0530
Subject: [PATCH 16/23] dev update

---
 v2/pkg/catalog/config/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go
index 8a996b62d..91b153cf2 100644
--- a/v2/pkg/catalog/config/config.go
+++ b/v2/pkg/catalog/config/config.go
@@ -26,7 +26,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.5.3`
+const Version = `2.5.4-dev`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()

From 1f8a9474cfa07682dc7e38a897b5b938a10a83f0 Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 25 Oct 2021 12:00:09 +0000
Subject: [PATCH 17/23] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25
 12:00:09 UTC 2021] :robot:

---
 SYNTAX-REFERENCE.md               | 25 ++++++++++++++++++++++
 nuclei-jsonschema.json            |  5 +++++
 v2/pkg/templates/templates_doc.go | 35 ++++++++++++++++++-------------
 3 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 30ed998fb..a3fffe351 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -2392,6 +2392,31 @@ read-size: 2048
 ```
 
 
+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>read-all</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+ReadAll determines if the data stream should be read till the end regardless of the size
+
+Default value for read-all is false.
+
+
+
+Examples:
+
+
+```yaml
+read-all: false
+```
+
+
 </div>
 
 <hr />
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 56253b7ed..57f8fc7fa 100755
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -809,6 +809,11 @@
           "title": "size of network response to read",
           "description": "Size of response to read at the end. Default is 1024 bytes"
         },
+        "read-all": {
+          "type": "boolean",
+          "title": "read all response stream",
+          "description": "Read all response stream till the server stops sending"
+        },
         "matchers": {
           "items": {
             "$ref": "#/definitions/matchers.Matcher"
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index 0b19e675d..d6042dfbd 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -843,7 +843,7 @@ func init() {
 			FieldName: "network",
 		},
 	}
-	NETWORKRequestDoc.Fields = make([]encoder.Doc, 9)
+	NETWORKRequestDoc.Fields = make([]encoder.Doc, 10)
 	NETWORKRequestDoc.Fields[0].Name = "id"
 	NETWORKRequestDoc.Fields[0].Type = "string"
 	NETWORKRequestDoc.Fields[0].Note = ""
@@ -883,22 +883,29 @@ func init() {
 	NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end"
 
 	NETWORKRequestDoc.Fields[5].AddExample("", 2048)
-	NETWORKRequestDoc.Fields[6].Name = "matchers"
-	NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher"
+	NETWORKRequestDoc.Fields[6].Name = "read-all"
+	NETWORKRequestDoc.Fields[6].Type = "bool"
 	NETWORKRequestDoc.Fields[6].Note = ""
-	NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
-	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[7].Name = "extractors"
-	NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+	NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false."
+	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size"
+
+	NETWORKRequestDoc.Fields[6].AddExample("", false)
+	NETWORKRequestDoc.Fields[7].Name = "matchers"
+	NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher"
 	NETWORKRequestDoc.Fields[7].Note = ""
-	NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
-	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[8].Name = "matchers-condition"
-	NETWORKRequestDoc.Fields[8].Type = "string"
+	NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[8].Name = "extractors"
+	NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor"
 	NETWORKRequestDoc.Fields[8].Note = ""
-	NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
-	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
-	NETWORKRequestDoc.Fields[8].Values = []string{
+	NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[9].Name = "matchers-condition"
+	NETWORKRequestDoc.Fields[9].Type = "string"
+	NETWORKRequestDoc.Fields[9].Note = ""
+	NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Values = []string{
 		"and",
 		"or",
 	}

From 4a59c3dd84d034c0711e4b480fbd7cc64657f1bb Mon Sep 17 00:00:00 2001
From: GitHub Action <action@github.com>
Date: Mon, 25 Oct 2021 12:09:59 +0000
Subject: [PATCH 18/23] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25
 12:09:59 UTC 2021] :robot:

---
 nuclei-jsonschema.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 57f8fc7fa..1db207d77 100755
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -850,6 +850,7 @@
       ],
       "properties": {
         "id": {
+          "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$",
           "type": "string",
           "title": "id of the template",
           "description": "The Unique ID for the template",

From ef3397b84de7a49569d2893700689de515a5715d Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Mon, 25 Oct 2021 17:42:01 +0530
Subject: [PATCH 19/23] Mark error for template syntax warning

---
 v2/pkg/parsers/parser.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go
index 83059baa7..7cd0c7b99 100644
--- a/v2/pkg/parsers/parser.go
+++ b/v2/pkg/parsers/parser.go
@@ -35,6 +35,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 	}
 
 	if validationError := validateTemplateFields(template); validationError != nil {
+		stats.Increment(SyntaxErrorStats)
 		return false, validationError
 	}
 

From ca6cc513021d2e6a85aeb879055bc158c482fea8 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Mon, 25 Oct 2021 17:47:39 +0530
Subject: [PATCH 20/23] Color change for warning disclaimer

---
 v2/internal/runner/banner.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go
index b75f09c33..56bf4ea4b 100644
--- a/v2/internal/runner/banner.go
+++ b/v2/internal/runner/banner.go
@@ -20,6 +20,6 @@ func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
-	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
+	gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

From c6445519ecee46b0195a499d6746bb0a6bf372af Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Mon, 25 Oct 2021 18:19:27 +0530
Subject: [PATCH 21/23] Fixed bug with github client and paths with no slash

---
 v2/pkg/reporting/trackers/github/github.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go
index 672f62ede..8d6cdb864 100644
--- a/v2/pkg/reporting/trackers/github/github.go
+++ b/v2/pkg/reporting/trackers/github/github.go
@@ -58,6 +58,9 @@ func New(options *Options) (*Integration, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "could not parse custom baseurl")
 		}
+		if !strings.HasSuffix(parsed.Path, "/") {
+			parsed.Path += "/"
+		}
 		client.BaseURL = parsed
 	}
 	return &Integration{client: client, options: options}, nil

From 4a29443752b9f158cab15359dc09a3691262fca5 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Mon, 25 Oct 2021 18:22:33 +0530
Subject: [PATCH 22/23] fix panic: Only generate curl command if request is not
 nil

---
 v2/pkg/protocols/http/request.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go
index 1a619c179..3451e5f3d 100644
--- a/v2/pkg/protocols/http/request.go
+++ b/v2/pkg/protocols/http/request.go
@@ -384,7 +384,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	}()
 
 	var curlCommand string
-	if !request.Unsafe && resp != nil && generatedRequest.request != nil {
+	if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil {
 		bodyBytes, _ := generatedRequest.request.BodyBytes()
 		resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 		command, _ := http2curl.GetCurlCommand(resp.Request)

From f0580298413c73e1647fb5c3e45610e95f799242 Mon Sep 17 00:00:00 2001
From: Ice3man543 <nizamulrana@gmail.com>
Date: Tue, 26 Oct 2021 16:29:00 +0530
Subject: [PATCH 23/23] Misc

---
 v2/cmd/integration-test/http.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go
index b2ecc816b..c8bc6acc1 100644
--- a/v2/cmd/integration-test/http.go
+++ b/v2/cmd/integration-test/http.go
@@ -41,8 +41,6 @@ type httpInteractshRequest struct{}
 func (h *httpInteractshRequest) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		httpDebugRequestDump(r)
-
 		value := r.Header.Get("url")
 		if value != "" {
 			if resp, _ := http.DefaultClient.Get(value); resp != nil {
@@ -53,7 +51,7 @@ func (h *httpInteractshRequest) Execute(filePath string) error {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug)
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
 	if err != nil {
 		return err
 	}