diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml
index ad2b95610..96a5a1a56 100644
--- a/.github/workflows/functional-test.yml
+++ b/.github/workflows/functional-test.yml
@@ -24,6 +24,7 @@ jobs:
- name: Functional Tests
env:
GH_ACTION: true
+ GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
run: |
chmod +x run.sh
bash run.sh ${{ matrix.os }}
diff --git a/README.md b/README.md
index 4b516c40d..632087b73 100644
--- a/README.md
+++ b/README.md
@@ -115,8 +115,8 @@ FILTERING:
-eid, -exclude-id string[] templates to exclude based on template ids (comma-separated, file)
-it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration
-et, -exclude-templates string[] template or template directory to exclude (comma-separated, file)
- -s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical
- -es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical
+ -s, -severity value[] templates to run based on severity. Possible values: info, low, medium, high, critical, unknown
+ -es, -exclude-severity value[] templates to exclude based on severity. Possible values: info, low, medium, high, critical, unknown
-pt, -type value[] templates to run based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
-ept, -exclude-type value[] templates to exclude based on protocol type. Possible values: dns, file, http, headless, network, workflow, ssl, websocket, whois
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index b42f2d68b..62efafcb1 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -633,6 +633,8 @@ Enum Values:
- high
- critical
+
+ - unknown
@@ -3218,6 +3220,34 @@ Steps is the list of actions to run for headless request
+
+
+descriptions: |
+ User-Agent is the type of user-agent to use for the request.
+
+
+
+
+
+
+
+custom_user_agent string
+
+
+
+
+description: |
+ If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
+
+
+
+
+
+
@@ -3430,6 +3460,48 @@ Enum Values:
+## userAgent.UserAgentHolder
+UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
+
+Appears in:
+
+
+- headless.Request.user_agent
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Enum Values:
+
+
+ - random
+
+ - off
+
+ - default
+
+ - custom
+
+
+
+
+
+
+
+
## ssl.Request
Request is a request for the SSL protocol
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index 4362b235d..756178107 100755
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -103,7 +103,8 @@
"low",
"medium",
"high",
- "critical"
+ "critical",
+ "unknown"
],
"type": "string",
"title": "severity of the template",
@@ -119,6 +120,16 @@
}
]
},
+ "userAgent.UserAgentHolder": {
+ "enum": [
+ "off",
+ "default",
+ "custom"
+ ],
+ "type": "string",
+ "title": "userAgent for the headless",
+ "description": "userAgent for the headless http request"
+ },
"extractors.Extractor": {
"required": [
"type"
@@ -531,6 +542,17 @@
"title": "list of actions for headless request",
"description": "List of actions to run for headless request"
},
+ "user_agent": {
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$ref": "#/definitions/userAgent.UserAgentHolder",
+ "title": "user agent for the headless request",
+ "description": "User agent for the headless request"
+ },
+ "custom_user_agent": {
+ "type": "string",
+ "title": "custom user agent for the headless request",
+ "description": "Custom user agent for the headless request"
+ },
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml
index ba272fcf3..481ef8526 100644
--- a/v2/.goreleaser.yml
+++ b/v2/.goreleaser.yml
@@ -3,36 +3,48 @@ before:
- go mod tidy
builds:
-- env:
- - CGO_ENABLED=0
- goos:
- - windows
- - linux
- - darwin
- goarch:
- - amd64
- - 386
- - arm
- - arm64
+- main: cmd/nuclei/main.go
+ binary: nuclei
+ id: nuclei-cli
+
+ env:
+ - CGO_ENABLED=0
+
+ goos: [windows,linux,darwin]
+ goarch: [amd64,386,arm,arm64]
ignore:
- goos: darwin
- goarch: '386'
+ goarch: 386
- goos: windows
- goarch: 'arm'
+ goarch: arm
- goos: windows
- goarch: 'arm64'
-
- binary: '{{ .ProjectName }}'
- main: cmd/nuclei/main.go
+ goarch: arm64
flags:
- -trimpath
+- main: cmd/cve-annotate/main.go
+ binary: cve-annotate
+ id: annotate
+
+ env:
+ - CGO_ENABLED=0
+
+ goos: [linux]
+ goarch: [amd64]
+
archives:
- format: zip
+ id: nuclei
+ builds: [nuclei-cli]
replacements:
darwin: macOS
+- format: zip
+ id: annotate
+ builds: [annotate]
+ name_template: "{{ .Binary }}"
+
checksum:
algorithm: sha256
diff --git a/v2/go.mod b/v2/go.mod
index 8f5e9be13..3beeee0aa 100644
--- a/v2/go.mod
+++ b/v2/go.mod
@@ -15,7 +15,7 @@ require (
github.com/go-rod/rod v0.102.0
github.com/gobwas/ws v1.1.0
github.com/google/go-github v17.0.0+incompatible
- github.com/itchyny/gojq v0.12.6
+ github.com/itchyny/gojq v0.12.7
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/karlseguin/ccache v2.0.3+incompatible
@@ -43,7 +43,7 @@ require (
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.3.0
github.com/segmentio/ksuid v1.0.4
- github.com/shirou/gopsutil/v3 v3.22.1
+ github.com/shirou/gopsutil/v3 v3.22.2
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.4.1
github.com/syndtr/goleveldb v1.0.0
@@ -54,7 +54,7 @@ require (
github.com/ysmood/gson v0.6.4 // indirect
github.com/ysmood/leakless v0.7.0 // indirect
go.uber.org/atomic v1.9.0
- go.uber.org/multierr v1.7.0
+ go.uber.org/multierr v1.8.0
go.uber.org/ratelimit v0.2.0
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
@@ -63,7 +63,7 @@ require (
moul.io/http2curl v1.0.0
)
-require github.com/aws/aws-sdk-go v1.43.7
+require github.com/aws/aws-sdk-go v1.43.9
require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e
@@ -155,7 +155,7 @@ require (
goftp.io/server/v2 v2.0.0 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/mod v0.4.2 // indirect
- golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
+ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
diff --git a/v2/go.sum b/v2/go.sum
index 6ba455592..83020183e 100644
--- a/v2/go.sum
+++ b/v2/go.sum
@@ -83,8 +83,8 @@ github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3st
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.43.7 h1:Gbs53KxXJWbO3txoVkevf56bhdDFqRisl7MQQ6581vc=
-github.com/aws/aws-sdk-go v1.43.7/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.43.9 h1:k1S/29Bp2QD5ZopnGzIn0Sp63yyt3WH1JRE2OOU3Aig=
+github.com/aws/aws-sdk-go v1.43.9/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -281,8 +281,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
-github.com/itchyny/gojq v0.12.6 h1:VjaFn59Em2wTxDNGcrRkDK9ZHMNa8IksOgL13sLL4d0=
-github.com/itchyny/gojq v0.12.6/go.mod h1:ZHrkfu7A+RbZLy5J1/JKpS4poEqrzItSTGDItqsfP0A=
+github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
+github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA=
@@ -505,8 +505,8 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4=
-github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
-github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
+github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
+github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -627,8 +627,9 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
+go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
@@ -814,11 +815,11 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go
index 6a8bc2751..7c9628333 100644
--- a/v2/internal/runner/update.go
+++ b/v2/internal/runner/update.go
@@ -23,6 +23,7 @@ import (
"github.com/google/go-github/github"
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
+ "golang.org/x/oauth2"
"github.com/projectdiscovery/folderutil"
"github.com/projectdiscovery/gologger"
@@ -223,7 +224,16 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
// getLatestReleaseFromGithub returns the latest release from GitHub
func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
- gitHubClient := github.NewClient(nil)
+ var tc *http.Client
+ if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
+ ctx := context.Background()
+ ts := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: token},
+ )
+ tc = oauth2.NewClient(ctx, ts)
+ }
+
+ gitHubClient := github.NewClient(tc)
release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
if err != nil {
diff --git a/v2/pkg/model/types/severity/severity.go b/v2/pkg/model/types/severity/severity.go
index 5c3b28150..d840d9558 100644
--- a/v2/pkg/model/types/severity/severity.go
+++ b/v2/pkg/model/types/severity/severity.go
@@ -24,6 +24,8 @@ const (
High
// name:critical
Critical
+ // name:unknown
+ Unknown
limit
)
@@ -33,6 +35,7 @@ var severityMappings = map[Severity]string{
Medium: "medium",
High: "high",
Critical: "critical",
+ Unknown: "unknown",
}
func GetSupportedSeverities() Severities {
diff --git a/v2/pkg/model/types/severity/severity_test.go b/v2/pkg/model/types/severity/severity_test.go
index 6ba472388..30ff4be94 100644
--- a/v2/pkg/model/types/severity/severity_test.go
+++ b/v2/pkg/model/types/severity/severity_test.go
@@ -26,7 +26,7 @@ func TestYamlUnmarshalFail(t *testing.T) {
func TestGetSupportedSeverities(t *testing.T) {
severities := GetSupportedSeverities()
- assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical})
+ assert.Equal(t, severities, Severities{Info, Low, Medium, High, Critical, Unknown})
}
func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) {
diff --git a/v2/pkg/model/types/userAgent/user_agent.go b/v2/pkg/model/types/userAgent/user_agent.go
new file mode 100644
index 000000000..998bb67a6
--- /dev/null
+++ b/v2/pkg/model/types/userAgent/user_agent.go
@@ -0,0 +1,95 @@
+package userAgent
+
+import (
+ "encoding/json"
+ "strings"
+
+ "github.com/alecthomas/jsonschema"
+ "github.com/pkg/errors"
+)
+
+type UserAgent int
+
+// name:UserAgent
+const (
+ // name:random
+ Random UserAgent = iota
+ // name:off
+ Off
+ // name:default
+ Default
+ // name:custom
+ Custom
+ limit
+)
+
+var userAgentMappings = map[UserAgent]string{
+ Random: "random",
+ Off: "off",
+ Default: "default",
+ Custom: "custom",
+}
+
+func GetSupportedUserAgentOptions() []UserAgent {
+ var result []UserAgent
+ for index := UserAgent(1); index < limit; index++ {
+ result = append(result, index)
+ }
+ return result
+}
+
+func toUserAgent(valueToMap string) (UserAgent, error) {
+ normalizedValue := normalizeValue(valueToMap)
+ for key, currentValue := range userAgentMappings {
+ if normalizedValue == currentValue {
+ return key, nil
+ }
+ }
+ return -1, errors.New("Invalid userAgent: " + valueToMap)
+}
+
+func normalizeValue(value string) string {
+ return strings.TrimSpace(strings.ToLower(value))
+}
+
+func (userAgent UserAgent) String() string {
+ return userAgentMappings[userAgent]
+}
+
+// UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes
+type UserAgentHolder struct {
+ Value UserAgent `mapping:"true"`
+}
+
+func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Type {
+ gotType := &jsonschema.Type{
+ Type: "string",
+ Title: "userAgent for the headless",
+ Description: "userAgent for the headless http request",
+ }
+ for _, userAgent := range GetSupportedUserAgentOptions() {
+ gotType.Enum = append(gotType.Enum, userAgent.String())
+ }
+ return gotType
+}
+
+func (userAgentHolder *UserAgentHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
+ var marshalledUserAgent string
+ if err := unmarshal(&marshalledUserAgent); err != nil {
+ return err
+ }
+ computedUserAgent, err := toUserAgent(marshalledUserAgent)
+ if err != nil {
+ return err
+ }
+ userAgentHolder.Value = computedUserAgent
+ return nil
+}
+
+func (userAgentHolder *UserAgentHolder) MarshalJSON() ([]byte, error) {
+ return json.Marshal(userAgentHolder.Value.String())
+}
+
+func (userAgentHolder UserAgentHolder) MarshalYAML() (interface{}, error) {
+ return userAgentHolder.Value.String(), nil
+}
diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go
index 6c1046b67..8d72f85bc 100644
--- a/v2/pkg/protocols/headless/engine/engine.go
+++ b/v2/pkg/protocols/headless/engine/engine.go
@@ -8,7 +8,6 @@ import (
"runtime"
"strings"
- "github.com/corpix/uarand"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/pkg/errors"
@@ -89,9 +88,6 @@ func New(options *types.Options) (*Browser, error) {
customAgent = parts[1]
}
}
- if customAgent == "" {
- customAgent = uarand.GetRandom()
- }
httpclient, err := newHttpClient(options)
if err != nil {
@@ -116,6 +112,16 @@ func MustDisableSandbox() bool {
return runtime.GOOS == "linux" && os.Geteuid() == 0
}
+// SetUserAgent sets custom user agent to the browser
+func (b *Browser) SetUserAgent(customUserAgent string) {
+ b.customAgent = customUserAgent
+}
+
+// UserAgent fetch the currently set custom user agent
+func (b *Browser) UserAgent() string {
+ return b.customAgent
+}
+
// Close closes the browser engine
func (b *Browser) Close() {
b.engine.Close()
diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go
index dc0174708..5133cfdd5 100644
--- a/v2/pkg/protocols/headless/headless.go
+++ b/v2/pkg/protocols/headless/headless.go
@@ -1,9 +1,11 @@
package headless
import (
+ "github.com/corpix/uarand"
"github.com/pkg/errors"
"github.com/projectdiscovery/fileutil"
+ useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
@@ -33,6 +35,15 @@ type Request struct {
// Steps is the list of actions to run for headless request
Steps []*engine.Action `yaml:"steps,omitempty" jsonschema:"title=list of actions for headless request,description=List of actions to run for headless request"`
+ // descriptions: |
+ // User-Agent is the type of user-agent to use for the request.
+ UserAgent useragent.UserAgentHolder `yaml:"user_agent,omitempty" jsonschema:"title=user agent for the headless request,description=User agent for the headless request"`
+
+ // description: |
+ // If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request.
+ CustomUserAgent string `yaml:"custom_user_agent,omitempty" jsonschema:"title=custom user agent for the headless request,description=Custom user agent for the headless request"`
+ compiledUserAgent string
+
// Operators for the current request go here.
operators.Operators `yaml:",inline,omitempty"`
CompiledOperators *operators.Operators `yaml:"-"`
@@ -90,6 +101,21 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
}
}
+ // Compile User-Agent
+ switch request.UserAgent.Value {
+ case useragent.Off:
+ request.compiledUserAgent = " "
+ case useragent.Default:
+ request.compiledUserAgent = ""
+ case useragent.Custom:
+ if request.CustomUserAgent == "" {
+ return errors.New("please set custom_user_agent in the template")
+ }
+ request.compiledUserAgent = request.CustomUserAgent
+ case useragent.Random:
+ request.compiledUserAgent = uarand.GetRandom()
+ }
+
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go
index 7422447e7..661790903 100644
--- a/v2/pkg/protocols/headless/request.go
+++ b/v2/pkg/protocols/headless/request.go
@@ -26,6 +26,9 @@ func (request *Request) Type() templateTypes.ProtocolType {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
+ if request.options.Browser.UserAgent() == "" {
+ request.options.Browser.SetUserAgent(request.compiledUserAgent)
+ }
payloads := generators.BuildPayloadFromOptions(request.options.Options)
if request.generator != nil {
iterator := request.generator.NewIterator()
diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go
index 53f9254ac..decc287d3 100644
--- a/v2/pkg/protocols/network/request.go
+++ b/v2/pkg/protocols/network/request.go
@@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
actualAddress := replacer.Replace(kv.address, variables)
if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
- gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
+ gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue
}
}
@@ -223,7 +223,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
default:
buf := make([]byte, bufferSize)
nBuf, err := conn.Read(buf)
- if err != nil && !os.IsTimeout(err) {
+ if err != nil && !os.IsTimeout(err) && err != io.EOF {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
closeTimer(readInterval)
return errors.Wrap(err, "could not read from server")
@@ -236,7 +236,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
} else {
final = make([]byte, bufferSize)
n, err = conn.Read(final)
- if err != nil && err != io.EOF {
+ if err != nil && !os.IsTimeout(err) && err != io.EOF {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
return errors.Wrap(err, "could not read from server")
}
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index dcb2739a0..f6cda7ad2 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -31,6 +31,7 @@ var (
HEADLESSRequestDoc encoder.Doc
ENGINEActionDoc encoder.Doc
ActionTypeHolderDoc encoder.Doc
+ USERAGENTUserAgentHolderDoc encoder.Doc
SSLRequestDoc encoder.Doc
WEBSOCKETRequestDoc encoder.Doc
WEBSOCKETInputDoc encoder.Doc
@@ -270,6 +271,7 @@ func init() {
"medium",
"high",
"critical",
+ "unknown",
}
MODELClassificationDoc.Type = "model.Classification"
@@ -1441,7 +1443,7 @@ func init() {
Value: "Headless response received from client (default)",
},
}
- HEADLESSRequestDoc.Fields = make([]encoder.Doc, 7)
+ HEADLESSRequestDoc.Fields = make([]encoder.Doc, 9)
HEADLESSRequestDoc.Fields[0].Name = "id"
HEADLESSRequestDoc.Fields[0].Type = "string"
HEADLESSRequestDoc.Fields[0].Note = ""
@@ -1462,22 +1464,32 @@ func init() {
HEADLESSRequestDoc.Fields[3].Note = ""
HEADLESSRequestDoc.Fields[3].Description = "Steps is the list of actions to run for headless request"
HEADLESSRequestDoc.Fields[3].Comments[encoder.LineComment] = "Steps is the list of actions to run for headless request"
- HEADLESSRequestDoc.Fields[4].Name = "matchers"
- HEADLESSRequestDoc.Fields[4].Type = "[]matchers.Matcher"
+ HEADLESSRequestDoc.Fields[4].Name = "user_agent"
+ HEADLESSRequestDoc.Fields[4].Type = "userAgent.UserAgentHolder"
HEADLESSRequestDoc.Fields[4].Note = ""
- HEADLESSRequestDoc.Fields[4].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
- HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
- HEADLESSRequestDoc.Fields[5].Name = "extractors"
- HEADLESSRequestDoc.Fields[5].Type = "[]extractors.Extractor"
+ HEADLESSRequestDoc.Fields[4].Description = "descriptions: |\n User-Agent is the type of user-agent to use for the request."
+ HEADLESSRequestDoc.Fields[4].Comments[encoder.LineComment] = " descriptions: |"
+ HEADLESSRequestDoc.Fields[5].Name = "custom_user_agent"
+ HEADLESSRequestDoc.Fields[5].Type = "string"
HEADLESSRequestDoc.Fields[5].Note = ""
- HEADLESSRequestDoc.Fields[5].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
- HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
- HEADLESSRequestDoc.Fields[6].Name = "matchers-condition"
- HEADLESSRequestDoc.Fields[6].Type = "string"
+ HEADLESSRequestDoc.Fields[5].Description = "description: |\n If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request."
+ HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = " description: |"
+ HEADLESSRequestDoc.Fields[6].Name = "matchers"
+ HEADLESSRequestDoc.Fields[6].Type = "[]matchers.Matcher"
HEADLESSRequestDoc.Fields[6].Note = ""
- HEADLESSRequestDoc.Fields[6].Description = "MatchersCondition is the condition between the matchers. Default is OR."
- HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
- HEADLESSRequestDoc.Fields[6].Values = []string{
+ HEADLESSRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+ HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+ HEADLESSRequestDoc.Fields[7].Name = "extractors"
+ HEADLESSRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+ HEADLESSRequestDoc.Fields[7].Note = ""
+ HEADLESSRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+ HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+ HEADLESSRequestDoc.Fields[8].Name = "matchers-condition"
+ HEADLESSRequestDoc.Fields[8].Type = "string"
+ HEADLESSRequestDoc.Fields[8].Note = ""
+ HEADLESSRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+ HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+ HEADLESSRequestDoc.Fields[8].Values = []string{
"and",
"or",
}
@@ -1553,6 +1565,28 @@ func init() {
"waitvisible",
}
+ USERAGENTUserAgentHolderDoc.Type = "userAgent.UserAgentHolder"
+ USERAGENTUserAgentHolderDoc.Comments[encoder.LineComment] = " UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
+ USERAGENTUserAgentHolderDoc.Description = "UserAgentHolder holds a UserAgent type. Required for un/marshalling purposes"
+ USERAGENTUserAgentHolderDoc.AppearsIn = []encoder.Appearance{
+ {
+ TypeName: "headless.Request",
+ FieldName: "user_agent",
+ },
+ }
+ USERAGENTUserAgentHolderDoc.Fields = make([]encoder.Doc, 1)
+ USERAGENTUserAgentHolderDoc.Fields[0].Name = ""
+ USERAGENTUserAgentHolderDoc.Fields[0].Type = "UserAgent"
+ USERAGENTUserAgentHolderDoc.Fields[0].Note = ""
+ USERAGENTUserAgentHolderDoc.Fields[0].Description = ""
+ USERAGENTUserAgentHolderDoc.Fields[0].Comments[encoder.LineComment] = ""
+ USERAGENTUserAgentHolderDoc.Fields[0].EnumFields = []string{
+ "random",
+ "off",
+ "default",
+ "custom",
+ }
+
SSLRequestDoc.Type = "ssl.Request"
SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol"
SSLRequestDoc.Description = "Request is a request for the SSL protocol"
@@ -1891,6 +1925,7 @@ func GetTemplateDoc() *encoder.FileDoc {
&HEADLESSRequestDoc,
&ENGINEActionDoc,
&ActionTypeHolderDoc,
+ &USERAGENTUserAgentHolderDoc,
&SSLRequestDoc,
&WEBSOCKETRequestDoc,
&WEBSOCKETInputDoc,
diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go
index f8bfcae9c..a681c286c 100644
--- a/v2/pkg/testutils/integration.go
+++ b/v2/pkg/testutils/integration.go
@@ -53,6 +53,9 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro
cmd.Args = append(cmd.Args, "-silent")
}
data, err := cmd.Output()
+ if debug {
+ fmt.Println(string(data))
+ }
if err != nil {
return nil, err
}
@@ -76,6 +79,9 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args
fmt.Println(cmd.String())
}
data, err := cmd.CombinedOutput()
+ if debug {
+ fmt.Println(string(data))
+ }
if err != nil {
return "", err
}