Merge branch 'dev' into issue-1594-regexp

.github/workflows/functional-test.yml

@@ -24,6 +24,7 @@ jobs:
       - name: Functional Tests
         env:
           GH_ACTION: true
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}

README.md

@@ -115,8 +115,8 @@ FILTERING:
    -eid, -exclude-id string[]        templates to exclude based on template ids (comma-separated, file)
    -it, -include-templates string[]  templates to be executed even if they are excluded either by default or configuration
    -et, -exclude-templates string[]  template or template directory to exclude (comma-separated, file)
-   -s, -severity value[]             templates to run based on severity. Possible values: info, low, medium, high, critical
+   -s, -severity value[]             templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
-   -es, -exclude-severity value[]    templates to exclude based on severity. Possible values: info, low, medium, high, critical
+   -es, -exclude-severity value[]    templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
    -pt, -type value[]                templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
    -ept, -exclude-type value[]       templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
 

SYNTAX-REFERENCE.md

@@ -633,6 +633,8 @@ Enum Values:
   - <code>high</code>
 
   - <code>critical</code>
+
+  - <code>unknown</code>
 </div>
 
 <hr />
@@ -3218,6 +3220,34 @@ Steps is the list of actions to run for headless request
 
 <div class="dd">
 
+<code>user_agent</code>  <i><a href="#useragentuseragentholder">userAgent.UserAgentHolder</a></i>
+
+</div>
+<div class="dt">
+
+descriptions: |
+ 	 User-Agent is the type of user-agent to use for the request.
+
+</div>
+
+<hr />
+
+<div class="dd">
+
+<code>custom_user_agent</code>  <i>string</i>
+
+</div>
+<div class="dt">
+
+description: |
+ 	 If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
+
+</div>
+
+<hr />
+
+<div class="dd">
+
 <code>matchers</code>  <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
 
 </div>
@@ -3430,6 +3460,48 @@ Enum Values:
 
 
 
+## userAgent.UserAgentHolder
+UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
+
+Appears in:
+
+
+- <code><a href="#headlessrequest">headless.Request</a>.user_agent</code>
+
+
+
+
+
+<hr />
+
+<div class="dd">
+
+<code></code>  <i>UserAgent</i>
+
+</div>
+<div class="dt">
+
+
+
+
+Enum Values:
+
+
+  - <code>random</code>
+
+  - <code>off</code>
+
+  - <code>default</code>
+
+  - <code>custom</code>
+</div>
+
+<hr />
+
+
+
+
+
 ## ssl.Request
 Request is a request for the SSL protocol
 

nuclei-jsonschema.json

@@ -103,7 +103,8 @@
         "low",
         "medium",
         "high",
-        "critical"
+        "critical",
+        "unknown"
       ],
       "type": "string",
       "title": "severity of the template",
@@ -119,6 +120,16 @@
         }
       ]
     },
+    "userAgent.UserAgentHolder": {
+      "enum": [
+        "off",
+        "default",
+        "custom"
+      ],
+      "type": "string",
+      "title": "userAgent for the headless",
+      "description": "userAgent for the headless http request"
+    },
     "extractors.Extractor": {
       "required": [
         "type"
@@ -531,6 +542,17 @@
           "title": "list of actions for headless request",
           "description": "List of actions to run for headless request"
         },
+        "user_agent": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/userAgent.UserAgentHolder",
+          "title": "user agent for the headless request",
+          "description": "User agent for the headless request"
+        },
+        "custom_user_agent": {
+          "type": "string",
+          "title": "custom user agent for the headless request",
+          "description": "Custom user agent for the headless request"
+        },
         "matchers": {
           "items": {
             "$ref": "#/definitions/matchers.Matcher"

v2/.goreleaser.yml

@@ -3,36 +3,48 @@ before:
     - go mod tidy
 
 builds:
-- env:
-  - CGO_ENABLED=0
-  goos:
-    - windows
-    - linux
-    - darwin
-  goarch:
-    - amd64
-    - 386
-    - arm
-    - arm64
 
+- main: cmd/nuclei/main.go
+  binary: nuclei
+  id: nuclei-cli
+
+  env:
+  - CGO_ENABLED=0
+
+  goos: [windows,linux,darwin]
+  goarch: [amd64,386,arm,arm64]
   ignore:
     - goos: darwin
-      goarch: '386'
+      goarch: 386
     - goos: windows
-      goarch: 'arm'
+      goarch: arm
     - goos: windows
-      goarch: 'arm64'
+      goarch: arm64
-
-  binary: '{{ .ProjectName }}'
-  main: cmd/nuclei/main.go
 
   flags:
     - -trimpath
 
+- main: cmd/cve-annotate/main.go
+  binary: cve-annotate
+  id: annotate
+
+  env:
+  - CGO_ENABLED=0
+
+  goos: [linux]
+  goarch: [amd64]
+
 archives:
 - format: zip
+  id: nuclei
+  builds: [nuclei-cli]
   replacements:
       darwin: macOS
 
+- format: zip
+  id: annotate
+  builds: [annotate]
+  name_template: "{{ .Binary }}"
+
 checksum:
   algorithm: sha256

v2/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/go-rod/rod v0.102.0
 	github.com/gobwas/ws v1.1.0
 	github.com/google/go-github v17.0.0+incompatible
-	github.com/itchyny/gojq v0.12.6
+	github.com/itchyny/gojq v0.12.7
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/karlseguin/ccache v2.0.3+incompatible
@@ -43,7 +43,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
 	github.com/segmentio/ksuid v1.0.4
-	github.com/shirou/gopsutil/v3 v3.22.1
+	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.4.1
 	github.com/syndtr/goleveldb v1.0.0
@@ -54,7 +54,7 @@ require (
 	github.com/ysmood/gson v0.6.4 // indirect
 	github.com/ysmood/leakless v0.7.0 // indirect
 	go.uber.org/atomic v1.9.0
-	go.uber.org/multierr v1.7.0
+	go.uber.org/multierr v1.8.0
 	go.uber.org/ratelimit v0.2.0
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
@@ -63,7 +63,7 @@ require (
 	moul.io/http2curl v1.0.0
 )
 
-require github.com/aws/aws-sdk-go v1.43.7
+require github.com/aws/aws-sdk-go v1.43.9
 
 require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
 
@@ -155,7 +155,7 @@ require (
 	goftp.io/server/v2 v2.0.0 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

v2/go.sum

@@ -83,8 +83,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.43.7 h1:Gbs53KxXJWbO3txoVkevf56bhdDFqRisl7MQQ6581vc=
+github.com/aws/aws-sdk-go v1.43.9 h1:k1S/29Bp2QD5ZopnGzIn0Sp63yyt3WH1JRE2OOU3Aig=
-github.com/aws/aws-sdk-go v1.43.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.43.9/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -281,8 +281,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
 github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
-github.com/itchyny/gojq v0.12.6 h1:VjaFn59Em2wTxDNGcrRkDK9ZHMNa8IksOgL13sLL4d0=
+github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
-github.com/itchyny/gojq v0.12.6/go.mod h1:ZHrkfu7A+RbZLy5J1/JKpS4poEqrzItSTGDItqsfP0A=
+github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
 github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
 github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
@@ -505,8 +505,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
-github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
+github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
-github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
+github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -627,8 +627,9 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
 go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
 go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
@@ -814,11 +815,11 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

v2/internal/runner/update.go

@@ -23,6 +23,7 @@ import (
 	"github.com/google/go-github/github"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
 
 	"github.com/projectdiscovery/folderutil"
 	"github.com/projectdiscovery/gologger"
@@ -223,7 +224,16 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
 
 // getLatestReleaseFromGithub returns the latest release from GitHub
 func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
-	gitHubClient := github.NewClient(nil)
+	var tc *http.Client
+	if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
+		ctx := context.Background()
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: token},
+		)
+		tc = oauth2.NewClient(ctx, ts)
+	}
+
+	gitHubClient := github.NewClient(tc)
 
 	release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
 	if err != nil {

v2/pkg/model/types/severity/severity.go

@@ -24,6 +24,8 @@ const (
 	High
 	// name:critical
 	Critical
+	// name:unknown
+	Unknown
 	limit
 )
 
@@ -33,6 +35,7 @@ var severityMappings = map[Severity]string{
 	Medium:   "medium",
 	High:     "high",
 	Critical: "critical",
+	Unknown:  "unknown",
 }
 
 func GetSupportedSeverities() Severities {

v2/pkg/model/types/severity/severity_test.go

@@ -26,7 +26,7 @@ func TestYamlUnmarshalFail(t *testing.T) {
 
 func TestGetSupportedSeverities(t *testing.T) {
 	severities := GetSupportedSeverities()
-	assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
+	assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical, Unknown})
 }
 
 func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {

v2/pkg/model/types/userAgent/user_agent.go

@@ -0,0 +1,95 @@
+package userAgent
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+	"github.com/pkg/errors"
+)
+
+type UserAgent int
+
+// name:UserAgent
+const (
+	// name:random
+	Random UserAgent = iota
+	// name:off
+	Off
+	// name:default
+	Default
+	// name:custom
+	Custom
+	limit
+)
+
+var userAgentMappings = map[UserAgent]string{
+	Random:  "random",
+	Off:     "off",
+	Default: "default",
+	Custom:  "custom",
+}
+
+func GetSupportedUserAgentOptions() []UserAgent {
+	var result []UserAgent
+	for index := UserAgent(1); index < limit; index++ {
+		result = append(result, index)
+	}
+	return result
+}
+
+func toUserAgent(valueToMap string) (UserAgent, error) {
+	normalizedValue := normalizeValue(valueToMap)
+	for key, currentValue := range userAgentMappings {
+		if normalizedValue == currentValue {
+			return key, nil
+		}
+	}
+	return -1, errors.New("Invalid userAgent: " + valueToMap)
+}
+
+func normalizeValue(value string) string {
+	return strings.TrimSpace(strings.ToLower(value))
+}
+
+func (userAgent UserAgent) String() string {
+	return userAgentMappings[userAgent]
+}
+
+// UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
+type UserAgentHolder struct {
+	Value UserAgent `mapping:"true"`
+}
+
+func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Type {
+	gotType := &jsonschema.Type{
+		Type:        "string",
+		Title:       "userAgent for the headless",
+		Description: "userAgent for the headless http request",
+	}
+	for _, userAgent := range GetSupportedUserAgentOptions() {
+		gotType.Enum = append(gotType.Enum, userAgent.String())
+	}
+	return gotType
+}
+
+func (userAgentHolder *UserAgentHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var marshalledUserAgent string
+	if err := unmarshal(&marshalledUserAgent); err != nil {
+		return err
+	}
+	computedUserAgent, err := toUserAgent(marshalledUserAgent)
+	if err != nil {
+		return err
+	}
+	userAgentHolder.Value = computedUserAgent
+	return nil
+}
+
+func (userAgentHolder *UserAgentHolder) MarshalJSON() ([]byte, error) {
+	return json.Marshal(userAgentHolder.Value.String())
+}
+
+func (userAgentHolder UserAgentHolder) MarshalYAML() (interface{}, error) {
+	return userAgentHolder.Value.String(), nil
+}

v2/pkg/protocols/headless/engine/engine.go

@@ -8,7 +8,6 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/corpix/uarand"
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/launcher"
 	"github.com/pkg/errors"
@@ -89,9 +88,6 @@ func New(options *types.Options) (*Browser, error) {
 			customAgent = parts[1]
 		}
 	}
-	if customAgent == "" {
-		customAgent = uarand.GetRandom()
-	}
 
 	httpclient, err := newHttpClient(options)
 	if err != nil {
@@ -116,6 +112,16 @@ func MustDisableSandbox() bool {
 	return runtime.GOOS == "linux" && os.Geteuid() == 0
 }
 
+// SetUserAgent sets custom user agent to the browser
+func (b *Browser) SetUserAgent(customUserAgent string) {
+	b.customAgent = customUserAgent
+}
+
+// UserAgent fetch the currently set custom user agent
+func (b *Browser) UserAgent() string {
+	return b.customAgent
+}
+
 // Close closes the browser engine
 func (b *Browser) Close() {
 	b.engine.Close()

v2/pkg/protocols/headless/headless.go

@@ -1,9 +1,11 @@
 package headless
 
 import (
+	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/fileutil"
+	useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent"
 	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
@@ -33,6 +35,15 @@ type Request struct {
 	//   Steps is the list of actions to run for headless request
 	Steps []*engine.Action `yaml:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"`
 
+	// descriptions: |
+	// 	 User-Agent is the type of user-agent to use for the request.
+	UserAgent useragent.UserAgentHolder `yaml:"user_agent,omitempty" jsonschema:"title=user agent for the headless request,description=User agent for the headless request"`
+
+	// description: |
+	// 	 If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
+	CustomUserAgent   string `yaml:"custom_user_agent,omitempty" jsonschema:"title=custom user agent for the headless request,description=Custom user agent for the headless request"`
+	compiledUserAgent string
+
 	// Operators for the current request go here.
 	operators.Operators `yaml:",inline,omitempty"`
 	CompiledOperators   *operators.Operators `yaml:"-"`
@@ -90,6 +101,21 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 		}
 	}
 
+	// Compile User-Agent
+	switch request.UserAgent.Value {
+	case useragent.Off:
+		request.compiledUserAgent = " "
+	case useragent.Default:
+		request.compiledUserAgent = ""
+	case useragent.Custom:
+		if request.CustomUserAgent == "" {
+			return errors.New("please set custom_user_agent in the template")
+		}
+		request.compiledUserAgent = request.CustomUserAgent
+	case useragent.Random:
+		request.compiledUserAgent = uarand.GetRandom()
+	}
+
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
 		if err := compiled.Compile(); err != nil {

v2/pkg/protocols/headless/request.go

@@ -26,6 +26,9 @@ func (request *Request) Type() templateTypes.ProtocolType {
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
+	if request.options.Browser.UserAgent() == "" {
+		request.options.Browser.SetUserAgent(request.compiledUserAgent)
+	}
 	payloads := generators.BuildPayloadFromOptions(request.options.Options)
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()

v2/pkg/protocols/network/request.go

@@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
 		actualAddress := replacer.Replace(kv.address, variables)
 
 		if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
-			gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
+			gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
 			continue
 		}
 	}
@@ -223,7 +223,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 			default:
 				buf := make([]byte, bufferSize)
 				nBuf, err := conn.Read(buf)
-				if err != nil && !os.IsTimeout(err) {
+				if err != nil && !os.IsTimeout(err) && err != io.EOF {
 					request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 					closeTimer(readInterval)
 					return errors.Wrap(err, "could not read from server")
@@ -236,7 +236,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	} else {
 		final = make([]byte, bufferSize)
 		n, err = conn.Read(final)
-		if err != nil && err != io.EOF {
+		if err != nil && !os.IsTimeout(err) && err != io.EOF {
 			request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 			return errors.Wrap(err, "could not read from server")
 		}

v2/pkg/templates/templates_doc.go

@@ -31,6 +31,7 @@ var (
 	HEADLESSRequestDoc            encoder.Doc
 	ENGINEActionDoc               encoder.Doc
 	ActionTypeHolderDoc           encoder.Doc
+	USERAGENTUserAgentHolderDoc   encoder.Doc
 	SSLRequestDoc                 encoder.Doc
 	WEBSOCKETRequestDoc           encoder.Doc
 	WEBSOCKETInputDoc             encoder.Doc
@@ -270,6 +271,7 @@ func init() {
 		"medium",
 		"high",
 		"critical",
+		"unknown",
 	}
 
 	MODELClassificationDoc.Type = "model.Classification"
@@ -1441,7 +1443,7 @@ func init() {
 			Value: "Headless response received from client (default)",
 		},
 	}
-	HEADLESSRequestDoc.Fields = make([]encoder.Doc, 7)
+	HEADLESSRequestDoc.Fields = make([]encoder.Doc, 9)
 	HEADLESSRequestDoc.Fields[0].Name = "id"
 	HEADLESSRequestDoc.Fields[0].Type = "string"
 	HEADLESSRequestDoc.Fields[0].Note = ""
@@ -1462,22 +1464,32 @@ func init() {
 	HEADLESSRequestDoc.Fields[3].Note = ""
 	HEADLESSRequestDoc.Fields[3].Description = "Steps is the list of actions to run for headless request"
 	HEADLESSRequestDoc.Fields[3].Comments[encoder.LineComment] = "Steps is the list of actions to run for headless request"
-	HEADLESSRequestDoc.Fields[4].Name = "matchers"
+	HEADLESSRequestDoc.Fields[4].Name = "user_agent"
-	HEADLESSRequestDoc.Fields[4].Type = "[]matchers.Matcher"
+	HEADLESSRequestDoc.Fields[4].Type = "userAgent.UserAgentHolder"
 	HEADLESSRequestDoc.Fields[4].Note = ""
-	HEADLESSRequestDoc.Fields[4].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+	HEADLESSRequestDoc.Fields[4].Description = "descriptions: |\n 	 User-Agent is the type of user-agent to use for the request."
-	HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+	HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = " descriptions: |"
-	HEADLESSRequestDoc.Fields[5].Name = "extractors"
+	HEADLESSRequestDoc.Fields[5].Name = "custom_user_agent"
-	HEADLESSRequestDoc.Fields[5].Type = "[]extractors.Extractor"
+	HEADLESSRequestDoc.Fields[5].Type = "string"
 	HEADLESSRequestDoc.Fields[5].Note = ""
-	HEADLESSRequestDoc.Fields[5].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	HEADLESSRequestDoc.Fields[5].Description = "description: |\n 	 If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request."
-	HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = " description: |"
-	HEADLESSRequestDoc.Fields[6].Name = "matchers-condition"
+	HEADLESSRequestDoc.Fields[6].Name = "matchers"
-	HEADLESSRequestDoc.Fields[6].Type = "string"
+	HEADLESSRequestDoc.Fields[6].Type = "[]matchers.Matcher"
 	HEADLESSRequestDoc.Fields[6].Note = ""
-	HEADLESSRequestDoc.Fields[6].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
-	HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
-	HEADLESSRequestDoc.Fields[6].Values = []string{
+	HEADLESSRequestDoc.Fields[7].Name = "extractors"
+	HEADLESSRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+	HEADLESSRequestDoc.Fields[7].Note = ""
+	HEADLESSRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+	HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+	HEADLESSRequestDoc.Fields[8].Name = "matchers-condition"
+	HEADLESSRequestDoc.Fields[8].Type = "string"
+	HEADLESSRequestDoc.Fields[8].Note = ""
+	HEADLESSRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+	HEADLESSRequestDoc.Fields[8].Values = []string{
 		"and",
 		"or",
 	}
@@ -1553,6 +1565,28 @@ func init() {
 		"waitvisible",
 	}
 
+	USERAGENTUserAgentHolderDoc.Type = "userAgent.UserAgentHolder"
+	USERAGENTUserAgentHolderDoc.Comments[encoder.LineComment] = " UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
+	USERAGENTUserAgentHolderDoc.Description = "UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
+	USERAGENTUserAgentHolderDoc.AppearsIn = []encoder.Appearance{
+		{
+			TypeName:  "headless.Request",
+			FieldName: "user_agent",
+		},
+	}
+	USERAGENTUserAgentHolderDoc.Fields = make([]encoder.Doc, 1)
+	USERAGENTUserAgentHolderDoc.Fields[0].Name = ""
+	USERAGENTUserAgentHolderDoc.Fields[0].Type = "UserAgent"
+	USERAGENTUserAgentHolderDoc.Fields[0].Note = ""
+	USERAGENTUserAgentHolderDoc.Fields[0].Description = ""
+	USERAGENTUserAgentHolderDoc.Fields[0].Comments[encoder.LineComment] = ""
+	USERAGENTUserAgentHolderDoc.Fields[0].EnumFields = []string{
+		"random",
+		"off",
+		"default",
+		"custom",
+	}
+
 	SSLRequestDoc.Type = "ssl.Request"
 	SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol"
 	SSLRequestDoc.Description = "Request is a request for the SSL protocol"
@@ -1891,6 +1925,7 @@ func GetTemplateDoc() *encoder.FileDoc {
 			&HEADLESSRequestDoc,
 			&ENGINEActionDoc,
 			&ActionTypeHolderDoc,
+			&USERAGENTUserAgentHolderDoc,
 			&SSLRequestDoc,
 			&WEBSOCKETRequestDoc,
 			&WEBSOCKETInputDoc,

v2/pkg/testutils/integration.go

@@ -53,6 +53,9 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
 		cmd.Args = append(cmd.Args, "-silent")
 	}
 	data, err := cmd.Output()
+	if debug {
+		fmt.Println(string(data))
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -76,6 +79,9 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args
 		fmt.Println(cmd.String())
 	}
 	data, err := cmd.CombinedOutput()
+	if debug {
+		fmt.Println(string(data))
+	}
 	if err != nil {
 		return "", err
 	}