Merge branch 'dev' of https://github.com/projectdiscovery/nuclei into more-protocols

README.md

@@ -90,30 +90,29 @@ TARGET:
    -l, -list string      path to file containing a list of target URLs/hosts to scan (one per line)
 
 TEMPLATES:
-   -tl                      list all available templates
    -t, -templates string[]  template or template directory paths to include in the scan
-   -w, -workflows string[]  list of workflows to run
+   -nt, -new-templates      run only new templates added in latest nuclei-templates release
-   -nt, -new-templates      run newly added templates only
+   -w, -workflows string[]  workflow or workflow directory paths to include in the scan
    -validate                validate the passed templates to nuclei
+   -tl                      list all available templates
 
 FILTERING:
-   -tags string[]                         execute a subset of templates that contain the provided tags
+   -tags string[]                    execute a subset of templates that contain the provided tags
-   -include-tags string[]                 tags from the default deny list that permit executing more intrusive templates
+   -etags, -exclude-tags string[]    exclude templates with the provided tags
-   -etags, -exclude-tags string[]         exclude templates with the provided tags
+   -itags, -include-tags string[]    tags from the default deny list that permit executing more intrusive templates
-   -include-templates string[]            templates to be executed even if they are excluded either by default or configuration
+   -et, -exclude-templates string[]  template or template directory paths to exclude
-   -exclude-templates, -exclude string[]  template or template directory paths to exclude
+   -it, -include-templates string[]  templates to be executed even if they are excluded either by default or configuration
-   -severity, -impact value[]             Templates to run based on severity. Possible values: info, low, medium, high, critical
+   -s, -severity value[]             Templates to run based on severity. Possible values - info,low,medium,high,critical
-   -author string[]                       execute templates that are (co-)created by the specified authors
+   -es, -exclude-severity value[]    Templates to exclude based on severity. Possible values - info,low,medium,high,critical
+   -a, -author string[]              execute templates that are (co-)created by the specified authors
 
 OUTPUT:
    -o, -output string            output file to write found issues/vulnerabilities
    -silent                       display findings only
-   -v, -verbose                  show verbose output
-   -vv                           display extra verbose information
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
-   -nm, -no-meta                 don't display match metadata in CLI output
+   -nm, -no-meta                 don't display match metadata
    -nts, -no-timestamp           don't display timestamp metadata in CLI output
    -rdb, -report-db string       local nuclei reporting database (always use this to persist report data)
    -me, -markdown-export string  directory to export results in markdown format
@@ -125,37 +124,39 @@ CONFIGURATIONS:
    -H, -header string[]        custom headers in header:value format
    -V, -var value              custom vars in var=value format
    -r, -resolvers string       file containing resolver list for nuclei
-   -system-resolvers           use system DNS resolving as error fallback
+   -sr, -system-resolvers      use system DNS resolving as error fallback
    -passive                    enable passive HTTP response processing mode
-   -env-vars                   enable environment variables support
+   -ev, -env-vars              enable environment variables to be used in template
 
 INTERACTSH:
-   -no-interactsh                     disable interactsh server for OOB testing
+   -iserver, -interactsh-server string  interactsh server url for self-hosted instance (default "https://interactsh.com")
-   -interactsh-url string             interactsh server url for self-hosted instance (default "https://interactsh.com")
+   -itoken, -interactsh-token string    authentication token for self-hosted interactsh server
-   -interactsh-token string           authentication token for self-hosted interactsh server
+   -interactions-cache-size int         number of requests to keep in the interactions cache (default 5000)
-   -interactions-cache-size int       number of requests to keep in the interactions cache (default 5000)
+   -interactions-eviction int           number of seconds to wait before evicting requests from cache (default 60)
-   -interactions-eviction int         number of seconds to wait before evicting requests from cache (default 60)
+   -interactions-poll-duration int      number of seconds to wait before each interaction poll request (default 5)
-   -interactions-poll-duration int    number of seconds to wait before each interaction poll request (default 5)
+   -interactions-cooldown-period int    extra time for interaction polling before exiting (default 5)
-   -interactions-cooldown-period int  extra time for interaction polling before exiting (default 5)
+   -ni, -no-interactsh                  disable interactsh server for OAST testing, exclude OAST based templates
 
 RATE-LIMIT:
    -rl, -rate-limit int          maximum number of requests to send per second (default 150)
    -rlm, -rate-limit-minute int  maximum number of requests to send per minute
    -bs, -bulk-size int           maximum number of hosts to be analyzed in parallel per template (default 25)
-   -c, -concurrency int          maximum number of templates to be executed in parallel (default 10)
+   -c, -concurrency int          maximum number of templates to be executed in parallel (default 25)
 
 OPTIMIZATIONS:
    -timeout int               time to wait in seconds before timeout (default 5)
    -retries int               number of times to retry a failed request (default 1)
-   -max-host-error int        max errors for a host before skipping from scan (default 30)
+   -mhe, -max-host-error int  max errors for a host before skipping from scan (default 30)
    -project                   use a project folder to avoid sending same request multiple times
-   -project-path string       set a specific project path (default "$TMPDIR/")
+   -project-path string       set a specific project path
    -spm, -stop-at-first-path  stop processing HTTP requests after the first match (may break template/workflow logic)
+   -stream                    Stream mode - start elaborating without sorting the input
 
 HEADLESS:
-   -headless          enable templates that require headless browser support
+   -headless            enable templates that require headless browser support
-   -page-timeout int  seconds to wait for each page in headless mode (default 20)
+   -page-timeout int    seconds to wait for each page in headless mode (default 20)
-   -show-browser      show the browser on the screen when running templates with headless mode
+   -sb, -show-browser   show the browser on the screen when running templates with headless mode
+   -sc, -system-chrome  Use local installed chrome browser instead of nuclei installed
 
 DEBUG:
    -debug                     show all requests and responses
@@ -163,22 +164,24 @@ DEBUG:
    -debug-resp                show all received responses
    -proxy, -proxy-url string  URL of the HTTP proxy server
    -proxy-socks-url string    URL of the SOCKS proxy server
-   -trace-log string          file to write sent requests trace log
+   -tlog, -trace-log string   file to write sent requests trace log
    -version                   show nuclei version
+   -v, -verbose               show verbose output
+   -vv                        display extra verbose information
    -tv, -templates-version    shows the version of the installed nuclei-templates
 
 UPDATE:
-   -update                        update nuclei to the latest released version
+   -update                        update nuclei engine to the latest released version
-   -ut, -update-templates         update the community templates to latest released version
+   -ut, -update-templates         update nuclei-templates to latest released version
-   -nut, -no-update-templates     do not check for nuclei-templates updates
+   -ud, -update-directory string  overwrite the default directory to install nuclei-templates
-   -ud, -update-directory string  overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates")
+   -duc, -disable-update-check    disable automatic nuclei/templates update check
 
 STATISTICS:
    -stats                    display statistics about the running scan
-   -stats-json               write statistics data to an output file in JSONL(ines) format
+   -sj, -stats-json          write statistics data to an output file in JSONL(ines) format
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default 5)
-   -metrics                  expose nuclei metrics on a port
+   -m, -metrics              expose nuclei metrics on a port
-   -metrics-port int         port to expose nuclei metrics on (default 9092)
+   -mp, -metrics-port int    port to expose nuclei metrics on (default 9092)
 ```
 
 ### Running Nuclei

SYNTAX-REFERENCE.md

@@ -2392,6 +2392,31 @@ read-size: 2048
 ```
 
 
+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>read-all</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+ReadAll determines if the data stream should be read till the end regardless of the size
+
+Default value for read-all is false.
+
+
+
+Examples:
+
+
+```yaml
+read-all: false
+```
+
+
 </div>
 
 <hr />

integration_tests/http/interactsh.yaml

@@ -0,0 +1,19 @@
+id: interactsh-integration-test
+
+info:
+  name: Interactsh Integration Test
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    headers:
+      url: 'http://{{interactsh-url}}'
+
+    matchers:
+      - type: word
+        part: interactsh_protocol # Confirms the HTTP Interaction
+        words:
+          - "http"

nuclei-jsonschema.json

@@ -809,6 +809,11 @@
           "title": "size of network response to read",
           "description": "Size of response to read at the end. Default is 1024 bytes"
         },
+        "read-all": {
+          "type": "boolean",
+          "title": "read all response stream",
+          "description": "Read all response stream till the server stops sending"
+        },
         "matchers": {
           "items": {
             "$ref": "#/definitions/matchers.Matcher"
@@ -845,6 +850,7 @@
       ],
       "properties": {
         "id": {
+          "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$",
           "type": "string",
           "title": "id of the template",
           "description": "The Unique ID for the template",

v2/cmd/integration-test/http.go

@@ -31,9 +31,36 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/raw-unsafe-request.yaml":    &httpRawUnsafeRequest{},
 	"http/request-condition.yaml":     &httpRequestCondition{},
 	"http/request-condition-new.yaml": &httpRequestCondition{},
+	"http/interactsh.yaml":            &httpInteractshRequest{},
 	"http/self-contained.yaml":        &httpRequestSelContained{},
 }
 
+type httpInteractshRequest struct{}
+
+// Executes executes a test case and returns an error if occurred
+func (h *httpInteractshRequest) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		value := r.Header.Get("url")
+		if value != "" {
+			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+				resp.Body.Close()
+			}
+		}
+	}))
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}
+
 type httpGetHeaders struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/cmd/nuclei/main.go

@@ -141,7 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"),
 		flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"),
-		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"),
+		flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"),
 		flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"),
 	)
 

v2/internal/runner/banner.go

@@ -20,6 +20,6 @@ func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
 
-	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
+	gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
-	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
+	gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }

v2/internal/runner/options.go

@@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) {
 	// Check if stdin pipe was given
 	options.Stdin = hasStdin()
 
-	// if VerboseVerbose is set, it implicitly enables the Verbose option as well
-	if options.VerboseVerbose {
-		options.Verbose = true
-	}
-
 	// Read the inputs and configure the logging
 	configureOutput(options)
 
@@ -127,7 +122,7 @@ func isValidURL(urlString string) bool {
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
-	if options.Verbose || options.VerboseVerbose {
+	if options.Verbose {
 		gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
 	}
 	if options.Debug {

v2/pkg/catalog/config/config.go

@@ -26,7 +26,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.5.3-dev`
+const Version = `2.5.4-dev`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()

v2/pkg/parsers/parser.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 
@@ -17,7 +18,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 )
 
-const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+const (
+	mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
+	invalidFieldFormatTemplate    = "invalid field format for '%s' (allowed format is %s)"
+)
 
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
 func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
@@ -30,12 +34,12 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
 		return false, nil
 	}
 
-	templateInfo := template.Info
+	if validationError := validateTemplateFields(template); validationError != nil {
-	if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		stats.Increment(SyntaxErrorStats)
 		return false, validationError
 	}
 
-	return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
+	return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags)
 }
 
 // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@@ -45,10 +49,8 @@ func LoadWorkflow(templatePath string) (bool, error) {
 		return false, templateParseError
 	}
 
-	templateInfo := template.Info
-
 	if len(template.Workflows) > 0 {
-		if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
+		if validationError := validateTemplateFields(template); validationError != nil {
 			return false, validationError
 		}
 		return true, nil
@@ -71,18 +73,29 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode
 	return match, err
 }
 
-func validateMandatoryInfoFields(info *model.Info) error {
+func validateTemplateFields(template *templates.Template) error {
-	if info == nil {
+	info := template.Info
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "info")
+
-	}
+	var errors []string
 
 	if utils.IsBlank(info.Name) {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "name")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name"))
 	}
 
 	if info.Authors.IsEmpty() {
-		return fmt.Errorf(mandatoryFieldMissingTemplate, "author")
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author"))
 	}
+
+	if template.ID == "" {
+		errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id"))
+	} else if !templateIDRegexp.MatchString(template.ID) {
+		errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()))
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ", "))
+	}
+
 	return nil
 }
 
@@ -90,6 +103,7 @@ var (
 	parsedTemplatesCache *cache.Templates
 	ShouldValidate       bool
 	fieldErrorRegexp     = regexp.MustCompile(`not found in`)
+	templateIDRegexp     = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`)
 )
 
 const (

v2/pkg/parsers/parser_test.go

@@ -0,0 +1,110 @@
+package parsers
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadTemplate(t *testing.T) {
+	origTemplatesCache := parsedTemplatesCache
+	defer func() { parsedTemplatesCache = origTemplatesCache }()
+
+	tt := []struct {
+		name        string
+		template    *templates.Template
+		templateErr error
+
+		expectedErr error
+	}{
+		{
+			name: "valid",
+			template: &templates.Template{
+				ID: "CVE-2021-27330",
+				Info: model.Info{
+					Name:    "Valid template",
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+		},
+		{
+			name:        "emptyTemplate",
+			template:    &templates.Template{},
+			expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"),
+		},
+		{
+			name: "emptyNameWithInvalidID",
+			template: &templates.Template{
+				ID: "invalid id",
+				Info: model.Info{
+					Authors: stringslice.StringSlice{Value: "Author"},
+				},
+			},
+			expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr)
+
+			tagFilter := filter.New(&filter.Config{})
+			success, err := LoadTemplate(tc.name, tagFilter, nil)
+			if tc.expectedErr == nil {
+				require.NoError(t, err)
+				require.True(t, success)
+			} else {
+				require.Equal(t, tc.expectedErr, err)
+				require.False(t, success)
+			}
+		})
+	}
+
+	t.Run("invalidTemplateID", func(t *testing.T) {
+		tt := []struct {
+			id      string
+			success bool
+		}{
+			{id: "A-B-C", success: true},
+			{id: "A-B-C-1", success: true},
+			{id: "CVE_2021_27330", success: true},
+			{id: "ABC DEF", success: false},
+			{id: "_-__AAA_", success: false},
+			{id: " CVE-2021-27330", success: false},
+			{id: "CVE-2021-27330 ", success: false},
+			{id: "CVE-2021-27330-", success: false},
+			{id: "-CVE-2021-27330-", success: false},
+			{id: "CVE-2021--27330", success: false},
+			{id: "CVE-2021+27330", success: false},
+		}
+		for i, tc := range tt {
+			name := fmt.Sprintf("regexp%d", i)
+			t.Run(name, func(t *testing.T) {
+				template := &templates.Template{
+					ID: tc.id,
+					Info: model.Info{
+						Name:    "Valid template",
+						Authors: stringslice.StringSlice{Value: "Author"},
+					},
+				}
+				parsedTemplatesCache.Store(name, template, nil)
+
+				tagFilter := filter.New(&filter.Config{})
+				success, err := LoadTemplate(name, tagFilter, nil)
+				if tc.success {
+					require.NoError(t, err)
+					require.True(t, success)
+				} else {
+					require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err)
+					require.False(t, success)
+				}
+			})
+		}
+	})
+}

v2/pkg/protocols/headless/engine/http_client.go

@@ -1,13 +1,18 @@
 package engine
 
 import (
+	"context"
 	"crypto/tls"
+	"fmt"
+	"net"
 	"net/http"
+	"net/http/cookiejar"
 	"net/url"
 	"time"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+	"golang.org/x/net/proxy"
 )
 
 // newhttpClient creates a new http client for headless communication with a timeout
@@ -28,7 +33,35 @@ func newhttpClient(options *types.Options) *http.Client {
 		if proxyURL, err := url.Parse(options.ProxyURL); err == nil {
 			transport.Proxy = http.ProxyURL(proxyURL)
 		}
+	} else if options.ProxySocksURL != "" {
+		var proxyAuth *proxy.Auth
+
+		socksURL, proxyErr := url.Parse(options.ProxySocksURL)
+		if proxyErr == nil {
+			proxyAuth = &proxy.Auth{}
+			proxyAuth.User = socksURL.User.Username()
+			proxyAuth.Password, _ = socksURL.User.Password()
+		}
+		dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct)
+		dc := dialer.(interface {
+			DialContext(ctx context.Context, network, addr string) (net.Conn, error)
+		})
+		if proxyErr == nil {
+			transport.DialContext = dc.DialContext
+		}
 	}
 
-	return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second}
+	jar, _ := cookiejar.New(nil)
+
+	httpclient := &http.Client{
+		Transport: transport,
+		Timeout:   time.Duration(options.Timeout*3) * time.Second,
+		Jar:       jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// the browser should follow redirects not us
+			return http.ErrUseLastResponse
+		},
+	}
+
+	return httpclient
 }

v2/pkg/protocols/headless/engine/rules.go

@@ -8,6 +8,9 @@ import (
 
 // routingRuleHandler handles proxy rule for actions related to request/response modification
 func (p *Page) routingRuleHandler(ctx *rod.Hijack) {
+	// usually browsers don't use chunked transfer encoding so we set the content-length nevertheless
+	ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body()))
+
 	for _, rule := range p.rules {
 		if rule.Part != "request" {
 			continue

v2/pkg/protocols/http/request.go

@@ -384,7 +384,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
 	}()
 
 	var curlCommand string
-	if !request.Unsafe && resp != nil && generatedRequest.request != nil {
+	if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil {
 		bodyBytes, _ := generatedRequest.request.BodyBytes()
 		resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes))
 		command, _ := http2curl.GetCurlCommand(resp.Request)

v2/pkg/protocols/network/network.go

@@ -60,6 +60,13 @@ type Request struct {
 	// examples:
 	//   - value: "2048"
 	ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"`
+	// description: |
+	//   ReadAll determines if the data stream should be read till the end regardless of the size
+	//
+	//   Default value for read-all is false.
+	// examples:
+	//   - value: false
+	ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"`
 
 	// description: |
 	//   SelfContained specifies if the request is self contained.

v2/pkg/protocols/network/request.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -200,13 +201,48 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
 	if request.ReadSize != 0 {
 		bufferSize = request.ReadSize
 	}
-	final := make([]byte, bufferSize)
+
-	n, err := conn.Read(final)
+	var (
-	if err != nil && err != io.EOF {
+		final []byte
-		request.options.Output.Request(request.options.TemplateID, address, "network", err)
+		n     int
-		return errors.Wrap(err, "could not read from server")
+	)
+
+	if request.ReadAll {
+		readInterval := time.NewTimer(time.Second * 1)
+		// stop the timer and drain the channel
+		closeTimer := func(t *time.Timer) {
+			if !t.Stop() {
+				<-t.C
+			}
+		}
+	read_socket:
+		for {
+			select {
+			case <-readInterval.C:
+				closeTimer(readInterval)
+				break read_socket
+			default:
+				buf := make([]byte, bufferSize)
+				nBuf, err := conn.Read(buf)
+				if err != nil && !os.IsTimeout(err) {
+					request.options.Output.Request(request.options.TemplateID, address, "network", err)
+					closeTimer(readInterval)
+					return errors.Wrap(err, "could not read from server")
+				}
+				responseBuilder.Write(buf[:nBuf])
+				final = append(final, buf...)
+				n += nBuf
+			}
+		}
+	} else {
+		final = make([]byte, bufferSize)
+		n, err = conn.Read(final)
+		if err != nil && err != io.EOF {
+			request.options.Output.Request(request.options.TemplateID, address, "network", err)
+			return errors.Wrap(err, "could not read from server")
+		}
+		responseBuilder.Write(final[:n])
 	}
-	responseBuilder.Write(final[:n])
 
 	response := responseBuilder.String()
 	outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)

v2/pkg/reporting/trackers/github/github.go

@@ -58,6 +58,9 @@ func New(options *Options) (*Integration, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "could not parse custom baseurl")
 		}
+		if !strings.HasSuffix(parsed.Path, "/") {
+			parsed.Path += "/"
+		}
 		client.BaseURL = parsed
 	}
 	return &Integration{client: client, options: options}, nil

v2/pkg/templates/templates.go

@@ -29,7 +29,7 @@ type Template struct {
 	// examples:
 	//   - name: ID Example
 	//     value: "\"CVE-2021-19520\""
-	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"`
+	ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
 	// description: |
 	//   Info contains metadata information about the template.
 	// examples:

v2/pkg/templates/templates_doc.go

@@ -843,7 +843,7 @@ func init() {
 			FieldName: "network",
 		},
 	}
-	NETWORKRequestDoc.Fields = make([]encoder.Doc, 9)
+	NETWORKRequestDoc.Fields = make([]encoder.Doc, 10)
 	NETWORKRequestDoc.Fields[0].Name = "id"
 	NETWORKRequestDoc.Fields[0].Type = "string"
 	NETWORKRequestDoc.Fields[0].Note = ""
@@ -883,22 +883,29 @@ func init() {
 	NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end"
 
 	NETWORKRequestDoc.Fields[5].AddExample("", 2048)
-	NETWORKRequestDoc.Fields[6].Name = "matchers"
+	NETWORKRequestDoc.Fields[6].Name = "read-all"
-	NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher"
+	NETWORKRequestDoc.Fields[6].Type = "bool"
 	NETWORKRequestDoc.Fields[6].Note = ""
-	NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+	NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false."
-	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size"
-	NETWORKRequestDoc.Fields[7].Name = "extractors"
+
-	NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+	NETWORKRequestDoc.Fields[6].AddExample("", false)
+	NETWORKRequestDoc.Fields[7].Name = "matchers"
+	NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher"
 	NETWORKRequestDoc.Fields[7].Note = ""
-	NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
-	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[8].Name = "matchers-condition"
+	NETWORKRequestDoc.Fields[8].Name = "extractors"
-	NETWORKRequestDoc.Fields[8].Type = "string"
+	NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor"
 	NETWORKRequestDoc.Fields[8].Note = ""
-	NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
-	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
-	NETWORKRequestDoc.Fields[8].Values = []string{
+	NETWORKRequestDoc.Fields[9].Name = "matchers-condition"
+	NETWORKRequestDoc.Fields[9].Type = "string"
+	NETWORKRequestDoc.Fields[9].Note = ""
+	NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	NETWORKRequestDoc.Fields[9].Values = []string{
 		"and",
 		"or",
 	}