External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:25:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into research-aws-signing

Browse Source
This commit is contained in:
mzack 2021-11-18 21:50:49 +01:00
commit b42f0d32ea
36 changed files with 1046 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
SYNTAX-REFERENCE.md
View File

@ -880,7 +880,7 @@ Valid values:
<div class="dd">
<code>method</code> <i>string</i>
<code>method</code> <i>HTTPMethodTypeHolder</i>
</div>
<div class="dt">
@ -1658,7 +1658,7 @@ name: cookie-extractor
<div class="dd">
<code>type</code> <i>string</i>
<code>type</code> <i>TypeHolder</i>
</div>
<div class="dt">
@ -2043,7 +2043,7 @@ name: '{{FQDN}}'
<div class="dd">
<code>type</code> <i>string</i>
<code>type</code> <i>DNSRequestTypeHolder</i>
</div>
<div class="dt">
@ -2125,6 +2125,43 @@ retries: 5
```
</div>
<hr />
<div class="dd">
<code>trace</code> <i>bool</i>
</div>
<div class="dt">
Trace performs a trace operation for the target.
</div>
<hr />
<div class="dd">
<code>trace-max-recursion</code> <i>int</i>
</div>
<div class="dt">
TraceMaxRecursion is the number of max recursion allowed for trace operations
Examples:
```yaml
# Use a retry of 100 to 150 generally
trace-max-recursion: 100
```
</div>
<hr />
@ -2609,7 +2646,7 @@ data: hex_decode('50494e47')
<div class="dd">
<code>type</code> <i>string</i>
<code>type</code> <i>NetworkInputTypeHolder</i>
</div>
<div class="dt">
@ -2842,7 +2879,7 @@ Description is the optional description of the headless action
<div class="dd">
<code>action</code> <i>string</i>
<code>action</code> <i>ActionTypeHolder</i>
</div>
<div class="dt">

165
nuclei-jsonschema.json
View File

@ -130,15 +130,8 @@
"description": "Name of the extractor"
},
"type": {
"enum": [
"regex",
"kval",
"json",
"xpath"
],
"type": "string",
"title": "type of the extractor",
"description": "Type of the extractor"
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/extractors.TypeHolder"
},
"regex": {
"items": {
@ -201,6 +194,17 @@
"additionalProperties": false,
"type": "object"
},
"extractors.TypeHolder": {
"enum": [
"regex",
"kval",
"xpath",
"json"
],
"type": "string",
"title": "type of the extractor",
"description": "Type of the extractor"
},
"matchers.Matcher": {
"required": [
"type"
@ -318,6 +322,22 @@
"title": "type of the attack",
"description": "Type of the attack"
},
"dns.DNSRequestTypeHolder": {
"enum": [
"A",
"NS",
"DS",
"CNAME",
"SOA",
"PTR",
"MX",
"TXT",
"AAAA"
],
"type": "string",
"title": "type of DNS request to make",
"description": "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"
},
"dns.Request": {
"properties": {
"matchers": {
@ -356,18 +376,8 @@
"description": "Name is the Hostname to make DNS request for"
},
"type": {
"enum": [
"A",
"NS",
"DS",
"CNAME",
"SOA",
"PTR",
"MX",
"TXT",
"AAAA"
],
"type": "string",
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/dns.DNSRequestTypeHolder",
"title": "type of dns request to make",
"description": "Type is the type of DNS request to make"
},
@ -389,6 +399,16 @@
"title": "retries for dns request",
"description": "Retries is the number of retries for the DNS request"
},
"trace": {
"type": "boolean",
"title": "trace operation",
"description": "Trace performs a trace operation for the target."
},
"trace-max-recursion": {
"type": "integer",
"title": "trace-max-recursion level for dns request",
"description": "TraceMaxRecursion is the number of max recursion allowed for trace operations"
},
"recursion": {
"type": "boolean",
"title": "recurse all servers",
@ -539,30 +559,8 @@
"description": "Description of the headless action"
},
"action": {
"enum": [
"navigate",
"script",
"click",
"rightclick",
"text",
"screenshot",
"time",
"select",
"files",
"waitload",
"getresource",
"extract",
"setmethod",
"addheader",
"setheader",
"deleteheader",
"setbody",
"waitevent",
"keyboard",
"debug",
"sleep"
],
"type": "string",
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/engine.ActionTypeHolder",
"title": "action to perform",
"description": "Type of actions to perform"
}
@ -570,6 +568,52 @@
"additionalProperties": false,
"type": "object"
},
"engine.ActionTypeHolder": {
"enum": [
"navigate",
"script",
"click",
"rightclick",
"text",
"screenshot",
"time",
"select",
"files",
"waitload",
"getresource",
"extract",
"set-method",
"addheader",
"setheader",
"deleteheader",
"setbody",
"waitevent",
"keyboard",
"debug",
"sleep",
"waitvisible"
],
"type": "string",
"title": "action to perform",
"description": "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"
},
"http.HTTPMethodTypeHolder": {
"enum": [
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"CONNECT",
"OPTIONS",
"TRACE",
"PATCH",
"PURGE"
],
"type": "string",
"title": "method is the HTTP request method",
"description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"
},
"http.Request": {
"properties": {
"matchers": {
@ -631,19 +675,8 @@
"description": "Attack is the type of payload combinations to perform"
},
"method": {
"enum": [
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"CONNECT",
"OPTIONS",
"TRACE",
"PATCH",
"PURGE"
],
"type": "string",
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/http.HTTPMethodTypeHolder",
"title": "method is the http request method",
"description": "Method is the HTTP Request Method"
},
@ -754,11 +787,8 @@
"description": "Data is the data to send as the input"
},
"type": {
"enum": [
"hex",
"text"
],
"type": "string",
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/network.NetworkInputTypeHolder",
"title": "type is the type of input data",
"description": "Type of input specified in data field"
},
@ -776,6 +806,15 @@
"additionalProperties": false,
"type": "object"
},
"network.NetworkInputTypeHolder": {
"enum": [
"hex",
"text"
],
"type": "string",
"title": "type is the type of input data",
"description": "description=Type of input specified in data field,enum=hex,enum=text"
},
"network.Request": {
"properties": {
"id": {

2
v2/go.mod
View File

@ -36,7 +36,7 @@ require (
github.com/projectdiscovery/interactsh v0.0.6
github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660
github.com/projectdiscovery/retryablehttp-go v1.0.2
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9
github.com/projectdiscovery/yamldoc-go v1.0.2

4
v2/go.sum
View File

@ -588,6 +588,7 @@ github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/d
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
@ -624,8 +625,9 @@ github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohT
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE=
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 h1:Ooa5htghPkdyfpzy6Y5KLdyv4w8ePZWmfzFSPQlJStQ=
github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660/go.mod h1:UfszkO3x+GLKVOpXB7boddJKbwNCr+tMPSkfgCSNhl4=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U=
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=

10
v2/pkg/operators/extractors/compile.go
View File

@ -10,13 +10,12 @@ import (
// CompileExtractors performs the initial setup operation on an extractor
func (e *Extractor) CompileExtractors() error {
var ok bool
// Set up the extractor type
e.extractorType, ok = ExtractorTypes[e.Type]
if !ok {
computedType, err := toExtractorTypes(e.GetType().String())
if err != nil {
return fmt.Errorf("unknown extractor type specified: %s", e.Type)
}
e.extractorType = computedType
// Compile the regexes
for _, regex := range e.Regex {
compiled, err := regexp.Compile(regex)
@ -25,7 +24,6 @@ func (e *Extractor) CompileExtractors() error {
}
e.regexCompiled = append(e.regexCompiled, compiled)
}
for i, kval := range e.KVal {
e.KVal[i] = strings.ToLower(kval)
}
@ -43,7 +41,7 @@ func (e *Extractor) CompileExtractors() error {
}
if e.CaseInsensitive {
if e.Type != "kval" {
if e.GetType() != KValExtractor {
return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type)
}
for i := range e.KVal {

105
v2/pkg/operators/extractors/extractor_types.go Normal file
View File

@ -0,0 +1,105 @@
package extractors
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// ExtractorType is the type of the extractor specified
type ExtractorType int
const (
// RegexExtractor extracts responses with regexes
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
//limit
limit
)
// extractorMappings is a table for conversion of extractor type from string.
var extractorMappings = map[ExtractorType]string{
RegexExtractor: "regex",
KValExtractor: "kval",
XPathExtractor: "xpath",
JSONExtractor: "json",
}
// GetType returns the type of the matcher
func (e *Extractor) GetType() ExtractorType {
return e.Type.ExtractorType
}
// GetSupportedExtractorTypes returns list of supported types
func GetSupportedExtractorTypes() []ExtractorType {
var result []ExtractorType
for index := ExtractorType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toExtractorTypes(valueToMap string) (ExtractorType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range extractorMappings {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid extractor type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t ExtractorType) String() string {
return extractorMappings[t]
}
// TypeHolder is used to hold internal type of the extractor
type TypeHolder struct {
ExtractorType ExtractorType
}
func (holder TypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type of the extractor",
Description: "Type of the extractor",
}
for _, types := range GetSupportedExtractorTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toExtractorTypes(marshalledTypes)
if err != nil {
return err
}
holder.ExtractorType = computedType
return nil
}
func (holder *TypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.ExtractorType.String())
}
func (holder TypeHolder) MarshalYAML() (interface{}, error) {
return holder.ExtractorType.String(), nil
}

29
v2/pkg/operators/extractors/extractors.go
View File

@ -21,7 +21,7 @@ type Extractor struct {
// - "kval"
// - "json"
// - "xpath"
Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"`
Type TypeHolder `json:"name,omitempty" yaml:"type"`
// extractorType is the internal type of the extractor
extractorType ExtractorType
@ -113,30 +113,3 @@ type Extractor struct {
// - true
CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"`
}
// ExtractorType is the type of the extractor specified
type ExtractorType = int
const (
// RegexExtractor extracts responses with regexes
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
)
// ExtractorTypes is a table for conversion of extractor type from string.
var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
"xpath": XPathExtractor,
"json": JSONExtractor,
}
// GetType returns the type of the matcher
func (e *Extractor) GetType() ExtractorType {
return e.extractorType
}

2
v2/pkg/protocols/common/executer/executer.go
View File

@ -47,8 +47,6 @@ func (e *Executer) Execute(input string) (bool, error) {
dynamicValues := make(map[string]interface{})
previous := make(map[string]interface{})
for _, req := range e.requests {
req := req
err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID()
if ID != "" {

80
v2/pkg/protocols/common/expressions/variables.go
View File

@ -10,47 +10,55 @@ var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7
// ContainsUnresolvedVariables returns an error with variable names if the passed
// input contains unresolved {{<pattern-here>}} variables.
func ContainsUnresolvedVariables(data string) error {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables found: ")
for i, match := range matches {
if len(match) < 2 {
continue
func ContainsUnresolvedVariables(items ...string) error {
for _, data := range items {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
errorString.WriteString(match[1])
if i != len(matches)-1 {
errorString.WriteString(",")
}
}
errorMessage := errorString.String()
return errors.New(errorMessage)
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables found: ")
func ContainsVariablesWithNames(data string, names map[string]interface{}) error {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables with values found: ")
for i, match := range matches {
if len(match) < 2 {
continue
}
matchName := match[1]
if _, ok := names[matchName]; !ok {
errorString.WriteString(matchName)
for i, match := range matches {
if len(match) < 2 {
continue
}
errorString.WriteString(match[1])
if i != len(matches)-1 {
errorString.WriteString(",")
}
}
errorMessage := errorString.String()
return errors.New(errorMessage)
}
errorMessage := errorString.String()
return errors.New(errorMessage)
return nil
}
func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error {
for _, data := range items {
matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return nil
}
errorString := &strings.Builder{}
errorString.WriteString("unresolved variables with values found: ")
for i, match := range matches {
if len(match) < 2 {
continue
}
matchName := match[1]
if _, ok := names[matchName]; !ok {
errorString.WriteString(matchName)
if i != len(matches)-1 {
errorString.WriteString(",")
}
}
}
errorMessage := errorString.String()
return errors.New(errorMessage)
}
return nil
}

38
v2/pkg/protocols/dns/dns.go
View File

@ -11,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/retryabledns"
@ -43,7 +44,7 @@ type Request struct {
// - "MX"
// - "TXT"
// - "AAAA"
RequestType string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"`
RequestType DNSRequestTypeHolder `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"`
// description: |
// Class is the class of the DNS request.
//
@ -62,6 +63,15 @@ type Request struct {
// - name: Use a retry of 3 to 5 generally
// value: 5
Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"`
// description: |
// Trace performs a trace operation for the target.
Trace bool `yaml:"trace,omitempty" jsonschema:"title=trace operation,description=Trace performs a trace operation for the target."`
// description: |
// TraceMaxRecursion is the number of max recursion allowed for trace operations
// examples:
// - name: Use a retry of 100 to 150 generally
// value: 100
TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty" jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"`
CompiledOperators *operators.Operators `yaml:"-"`
dnsClient *retryabledns.Client
@ -96,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
dnsClientOptions.Resolvers = request.Resolvers
}
// Create a dns client for the class
client, err := dnsclientpool.Get(options.Options, dnsClientOptions)
client, err := request.getDnsClient(options, nil)
if err != nil {
return errors.Wrap(err, "could not get dns client")
}
@ -111,10 +121,32 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
}
request.class = classToInt(request.Class)
request.options = options
request.question = questionTypeToInt(request.RequestType)
request.question = questionTypeToInt(request.RequestType.String())
return nil
}
func (request *Request) getDnsClient(options *protocols.ExecuterOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {
dnsClientOptions := &dnsclientpool.Configuration{
Retries: request.Retries,
}
if len(request.Resolvers) > 0 {
if len(request.Resolvers) > 0 {
for _, resolver := range request.Resolvers {
if expressions.ContainsUnresolvedVariables(resolver) != nil {
var err error
resolver, err = expressions.Evaluate(resolver, metadata)
if err != nil {
return nil, errors.Wrap(err, "could not resolve resolvers expressions")
}
dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver)
}
}
}
dnsClientOptions.Resolvers = request.Resolvers
}
return dnsclientpool.Get(options.Options, dnsClientOptions)
}
// Requests returns the total number of requests the YAML rule will perform
func (request *Request) Requests() int {
return 1

2
v2/pkg/protocols/dns/dns_test.go
View File

@ -27,7 +27,7 @@ func TestDNSCompileMake(t *testing.T) {
testutils.Init(options)
const templateID = "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,

110
v2/pkg/protocols/dns/dns_types.go Normal file
View File

@ -0,0 +1,110 @@
package dns
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// DNSRequestType is the type of the method specified
type DNSRequestType int
const (
A DNSRequestType = iota + 1
NS
DS
CNAME
SOA
PTR
MX
TXT
AAAA
//limit
limit
)
// DNSRequestTypeMapping is a table for conversion of method from string.
var DNSRequestTypeMapping = map[DNSRequestType]string{
A: "A",
NS: "NS",
DS: "DS",
CNAME: "CNAME",
SOA: "SOA",
PTR: "PTR",
MX: "MX",
TXT: "TXT",
AAAA: "AAAA",
}
// GetSupportedDNSRequestTypes returns list of supported types
func GetSupportedDNSRequestTypes() []DNSRequestType {
var result []DNSRequestType
for index := DNSRequestType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toDNSRequestTypes(valueToMap string) (DNSRequestType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range DNSRequestTypeMapping {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid DNS request type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToUpper(value))
}
func (t DNSRequestType) String() string {
return DNSRequestTypeMapping[t]
}
// DNSRequestTypeHolder is used to hold internal type of the DNS type
type DNSRequestTypeHolder struct {
DNSRequestType DNSRequestType
}
func (holder DNSRequestTypeHolder) String() string {
return holder.DNSRequestType.String()
}
func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type of DNS request to make",
Description: "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA",
}
for _, types := range GetSupportedDNSRequestTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *DNSRequestTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toDNSRequestTypes(marshalledTypes)
if err != nil {
return err
}
holder.DNSRequestType = computedType
return nil
}
func (holder *DNSRequestTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.DNSRequestType.String())
}
func (holder DNSRequestTypeHolder) MarshalYAML() (interface{}, error) {
return holder.DNSRequestType.String(), nil
}

19
v2/pkg/protocols/dns/operators.go
View File

@ -2,6 +2,8 @@ package dns
import (
"bytes"
"fmt"
"strings"
"time"
"github.com/miekg/dns"
@ -12,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/retryabledns"
)
// Match matches a generic data response against a given matcher
@ -73,7 +76,7 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (in
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, tracedata *retryabledns.TraceData) output.InternalEvent {
return output.InternalEvent{
"host": host,
"matched": matched,
@ -87,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
"trace": traceToString(tracedata, false),
}
}
@ -126,3 +130,16 @@ func questionToString(resourceRecords []dns.Question) string {
}
return buffer.String()
}
func traceToString(tracedata *retryabledns.TraceData, withSteps bool) string {
buffer := &bytes.Buffer{}
if tracedata != nil {
for i, dnsRecord := range tracedata.DNSData {
if withSteps {
buffer.WriteString(fmt.Sprintf("request %d to resolver %s:\n", i, strings.Join(dnsRecord.Resolver, ",")))
}
buffer.WriteString(dnsRecord.Raw)
}
}
return buffer.String()
}

26
v2/pkg/protocols/dns/operators_test.go
View File

@ -23,7 +23,7 @@ func TestResponseToDSLMap(t *testing.T) {
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,
@ -44,8 +44,8 @@ func TestResponseToDSLMap(t *testing.T) {
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 12, "could not get correct number of items in dsl map")
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil)
require.Len(t, event, 13, "could not get correct number of items in dsl map")
require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode")
}
@ -55,7 +55,7 @@ func TestDNSOperatorMatch(t *testing.T) {
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,
@ -76,7 +76,7 @@ func TestDNSOperatorMatch(t *testing.T) {
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil)
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
@ -143,7 +143,7 @@ func TestDNSOperatorMatch(t *testing.T) {
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "ONE.ONE.ONE.ONE."}})
event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE")
event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE", nil)
matcher := &matchers.Matcher{
Part: "raw",
@ -166,7 +166,7 @@ func TestDNSOperatorExtract(t *testing.T) {
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,
@ -187,12 +187,12 @@ func TestDNSOperatorExtract(t *testing.T) {
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil)
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -205,7 +205,7 @@ func TestDNSOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"rcode"},
}
err = extractor.CompileExtractors()
@ -223,7 +223,7 @@ func TestDNSMakeResult(t *testing.T) {
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,
@ -238,7 +238,7 @@ func TestDNSMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
@ -257,7 +257,7 @@ func TestDNSMakeResult(t *testing.T) {
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil)
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)

41
v2/pkg/protocols/dns/request.go
View File

@ -12,6 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/retryabledns"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
@ -40,6 +41,14 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return errors.Wrap(err, "could not build request")
}
dnsClient := request.dnsClient
if varErr := expressions.ContainsUnresolvedVariables(request.Resolvers...); varErr != nil {
if dnsClient, varErr = request.getDnsClient(request.options, metadata); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr)
return nil
}
}
requestString := compiledRequest.String()
if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr)
@ -51,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
}
// Send the request to the target servers
response, err := request.dnsClient.Do(compiledRequest)
response, err := dnsClient.Do(compiledRequest)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
@ -64,20 +73,33 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain)
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input)
// perform trace if necessary
var tracedata *retryabledns.TraceData
if request.Trace {
tracedata, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion)
if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, "dns", err)
}
}
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, tracedata)
for k, v := range previous {
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
// TODO: dynamic values are not supported yet
dumpResponse(event, request.options, response.String(), domain)
if request.Trace {
dumpTraceData(event, request.options, traceToString(tracedata, true), domain)
}
callback(event)
return nil
}
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) {
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
hexDump := false
@ -90,6 +112,19 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
}
}
func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, tracedata, domain string) {
cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse {
hexDump := false
if responsehighlighter.HasBinaryContent(tracedata) {
hexDump = true
tracedata = hex.Dump([]byte(tracedata))
}
highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, tracedata, cliOptions.NoColor, hexDump)
gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse)
}
}
// isURL tests a string to determine if it is a well-structured url or not.
func isURL(toTest string) bool {
if _, err := url.ParseRequestURI(toTest); err != nil {

4
v2/pkg/protocols/dns/request_test.go
View File

@ -20,7 +20,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
RequestType: "A",
RequestType: DNSRequestTypeHolder{DNSRequestType: A},
Class: "INET",
Retries: 5,
ID: templateID,
@ -35,7 +35,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

6
v2/pkg/protocols/file/operators_test.go
View File

@ -154,7 +154,7 @@ func TestFileOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -167,7 +167,7 @@ func TestFileOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"raw"},
}
err = extractor.CompileExtractors()
@ -250,7 +250,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
Matchers: matchers,
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

2
v2/pkg/protocols/file/request_test.go
View File

@ -37,7 +37,7 @@ func TestFileExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

108
v2/pkg/protocols/headless/engine/action.go
View File

@ -2,110 +2,6 @@ package engine
import "strings"
// ActionType defines the action type for a browser action
type ActionType int8
// Types to be executed by the user.
const (
// ActionNavigate performs a navigation to the specified URL
// URL can include nuclei payload data such as URL, Hostname, etc.
ActionNavigate ActionType = iota + 1
// ActionScript executes a JS snippet on the page.
ActionScript
// ActionClick performs the left-click action on an Element.
ActionClick
// ActionRightClick performs the right-click action on an Element.
ActionRightClick
// ActionTextInput performs an action for a text input
ActionTextInput
// ActionScreenshot performs the screenshot action writing to a file.
ActionScreenshot
// ActionTimeInput performs an action on a time input.
ActionTimeInput
// ActionSelectInput performs an action on a select input.
ActionSelectInput
// ActionFilesInput performs an action on a file input.
ActionFilesInput
// ActionWaitLoad waits for the page to stop loading.
ActionWaitLoad
// ActionGetResource performs a get resource action on an element
ActionGetResource
// ActionExtract performs an extraction on an element
ActionExtract
// ActionSetMethod sets the request method
ActionSetMethod
// ActionAddHeader adds a header to the request
ActionAddHeader
// ActionSetHeader sets a header in the request
ActionSetHeader
// ActionDeleteHeader deletes a header from the request
ActionDeleteHeader
// ActionSetBody sets the value of the request body
ActionSetBody
// ActionWaitEvent waits for a specific event.
ActionWaitEvent
// ActionKeyboard performs a keyboard action event on a page.
ActionKeyboard
// ActionDebug debug slows down headless and adds a sleep to each page.
ActionDebug
// ActionSleep executes a sleep for a specified duration
ActionSleep
// ActionWaitVisible waits until an element appears.
ActionWaitVisible
)
// ActionStringToAction converts an action from string to internal representation
var ActionStringToAction = map[string]ActionType{
"navigate": ActionNavigate,
"script": ActionScript,
"click": ActionClick,
"rightclick": ActionRightClick,
"text": ActionTextInput,
"screenshot": ActionScreenshot,
"time": ActionTimeInput,
"select": ActionSelectInput,
"files": ActionFilesInput,
"waitload": ActionWaitLoad,
"getresource": ActionGetResource,
"extract": ActionExtract,
"setmethod": ActionSetMethod,
"addheader": ActionAddHeader,
"setheader": ActionSetHeader,
"deleteheader": ActionDeleteHeader,
"setbody": ActionSetBody,
"waitevent": ActionWaitEvent,
"keyboard": ActionKeyboard,
"debug": ActionDebug,
"sleep": ActionSleep,
"waitvisible": ActionWaitVisible,
}
// ActionToActionString converts an action from internal representation to string
var ActionToActionString = map[ActionType]string{
ActionNavigate: "navigate",
ActionScript: "script",
ActionClick: "click",
ActionRightClick: "rightclick",
ActionTextInput: "text",
ActionScreenshot: "screenshot",
ActionTimeInput: "time",
ActionSelectInput: "select",
ActionFilesInput: "files",
ActionWaitLoad: "waitload",
ActionGetResource: "getresource",
ActionExtract: "extract",
ActionSetMethod: "set-method",
ActionAddHeader: "addheader",
ActionSetHeader: "setheader",
ActionDeleteHeader: "deleteheader",
ActionSetBody: "setbody",
ActionWaitEvent: "waitevent",
ActionKeyboard: "keyboard",
ActionDebug: "debug",
ActionSleep: "sleep",
ActionWaitVisible: "waitvisible",
}
// Action is an action taken by the browser to reach a navigation
//
// Each step that the browser executes is an action. Most navigations
@ -152,13 +48,13 @@ type Action struct {
// - "keyboard"
// - "debug"
// - "sleep"
ActionType string `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
}
// String returns the string representation of an action
func (a *Action) String() string {
builder := &strings.Builder{}
builder.WriteString(a.ActionType)
builder.WriteString(a.ActionType.String())
if a.Name != "" {
builder.WriteString(" Name:")
builder.WriteString(a.Name)

185
v2/pkg/protocols/headless/engine/action_types.go Normal file
View File

@ -0,0 +1,185 @@
package engine
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// ActionType defines the action type for a browser action
type ActionType int8
// Types to be executed by the user.
const (
// ActionNavigate performs a navigation to the specified URL
// URL can include nuclei payload data such as URL, Hostname, etc.
ActionNavigate ActionType = iota + 1
// ActionScript executes a JS snippet on the page.
ActionScript
// ActionClick performs the left-click action on an Element.
ActionClick
// ActionRightClick performs the right-click action on an Element.
ActionRightClick
// ActionTextInput performs an action for a text input
ActionTextInput
// ActionScreenshot performs the screenshot action writing to a file.
ActionScreenshot
// ActionTimeInput performs an action on a time input.
ActionTimeInput
// ActionSelectInput performs an action on a select input.
ActionSelectInput
// ActionFilesInput performs an action on a file input.
ActionFilesInput
// ActionWaitLoad waits for the page to stop loading.
ActionWaitLoad
// ActionGetResource performs a get resource action on an element
ActionGetResource
// ActionExtract performs an extraction on an element
ActionExtract
// ActionSetMethod sets the request method
ActionSetMethod
// ActionAddHeader adds a header to the request
ActionAddHeader
// ActionSetHeader sets a header in the request
ActionSetHeader
// ActionDeleteHeader deletes a header from the request
ActionDeleteHeader
// ActionSetBody sets the value of the request body
ActionSetBody
// ActionWaitEvent waits for a specific event.
ActionWaitEvent
// ActionKeyboard performs a keyboard action event on a page.
ActionKeyboard
// ActionDebug debug slows down headless and adds a sleep to each page.
ActionDebug
// ActionSleep executes a sleep for a specified duration
ActionSleep
// ActionWaitVisible waits until an element appears.
ActionWaitVisible
// limit
limit
)
// ActionStringToAction converts an action from string to internal representation
var ActionStringToAction = map[string]ActionType{
"navigate": ActionNavigate,
"script": ActionScript,
"click": ActionClick,
"rightclick": ActionRightClick,
"text": ActionTextInput,
"screenshot": ActionScreenshot,
"time": ActionTimeInput,
"select": ActionSelectInput,
"files": ActionFilesInput,
"waitload": ActionWaitLoad,
"getresource": ActionGetResource,
"extract": ActionExtract,
"setmethod": ActionSetMethod,
"addheader": ActionAddHeader,
"setheader": ActionSetHeader,
"deleteheader": ActionDeleteHeader,
"setbody": ActionSetBody,
"waitevent": ActionWaitEvent,
"keyboard": ActionKeyboard,
"debug": ActionDebug,
"sleep": ActionSleep,
"waitvisible": ActionWaitVisible,
}
// ActionToActionString converts an action from internal representation to string
var ActionToActionString = map[ActionType]string{
ActionNavigate: "navigate",
ActionScript: "script",
ActionClick: "click",
ActionRightClick: "rightclick",
ActionTextInput: "text",
ActionScreenshot: "screenshot",
ActionTimeInput: "time",
ActionSelectInput: "select",
ActionFilesInput: "files",
ActionWaitLoad: "waitload",
ActionGetResource: "getresource",
ActionExtract: "extract",
ActionSetMethod: "set-method",
ActionAddHeader: "addheader",
ActionSetHeader: "setheader",
ActionDeleteHeader: "deleteheader",
ActionSetBody: "setbody",
ActionWaitEvent: "waitevent",
ActionKeyboard: "keyboard",
ActionDebug: "debug",
ActionSleep: "sleep",
ActionWaitVisible: "waitvisible",
}
// GetSupportedActionTypes returns list of supported types
func GetSupportedActionTypes() []ActionType {
var result []ActionType
for index := ActionType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toActionTypes(valueToMap string) (ActionType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range ActionToActionString {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid action type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t ActionType) String() string {
return ActionToActionString[t]
}
// ActionTypeHolder is used to hold internal type of the action
type ActionTypeHolder struct {
ActionType ActionType
}
func (holder ActionTypeHolder) String() string {
return holder.ActionType.String()
}
func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "action to perform",
Description: "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep",
}
for _, types := range GetSupportedActionTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *ActionTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toActionTypes(marshalledTypes)
if err != nil {
return err
}
holder.ActionType = computedType
return nil
}
func (holder *ActionTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.ActionType.String())
}
func (holder ActionTypeHolder) MarshalYAML() (interface{}, error) {
return holder.ActionType.String(), nil
}

4
v2/pkg/protocols/headless/engine/page_actions.go
View File

@ -24,9 +24,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s
outData := make(map[string]string)
for _, act := range actions {
actionType := ActionStringToAction[act.ActionType]
switch actionType {
switch act.ActionType.ActionType {
case ActionNavigate:
err = p.NavigateURL(act, outData, baseURL)
case ActionScript:

119
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) {
</body>
</html>`
actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}}
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
@ -50,10 +50,11 @@ func TestActionScript(t *testing.T) {
t.Run("run-and-results", func(t *testing.T) {
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}},
}
testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
@ -63,10 +64,10 @@ func TestActionScript(t *testing.T) {
t.Run("hook", func(t *testing.T) {
actions := []*Action{
{ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}},
}
testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
@ -87,9 +88,9 @@ func TestActionClick(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -120,9 +121,9 @@ func TestActionRightClick(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -145,9 +146,9 @@ func TestActionTextInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -162,9 +163,9 @@ func TestActionTextInput(t *testing.T) {
func TestActionHeadersChange(t *testing.T) {
actions := []*Action{
{ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -189,9 +190,9 @@ func TestActionScreenshot(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "screenshot", Data: map[string]string{"to": "test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -214,9 +215,9 @@ func TestActionTimeInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -242,9 +243,9 @@ func TestActionSelectInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -265,9 +266,9 @@ func TestActionFilesInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -291,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -316,8 +317,8 @@ func TestActionGetResource(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -336,8 +337,8 @@ func TestActionExtract(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -355,8 +356,8 @@ func TestActionSetMethod(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -367,9 +368,9 @@ func TestActionSetMethod(t *testing.T) {
func TestActionAddHeader(t *testing.T) {
actions := []*Action{
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -386,11 +387,11 @@ func TestActionAddHeader(t *testing.T) {
func TestActionDeleteHeader(t *testing.T) {
actions := []*Action{
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}},
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}},
{ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -407,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) {
func TestActionSetBody(t *testing.T) {
actions := []*Action{
{ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -435,10 +436,10 @@ func TestActionKeyboard(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "click", Data: map[string]string{"selector": "input"}},
{ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -461,8 +462,8 @@ func TestActionSleep(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "sleep", Data: map[string]string{"duration": "2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -484,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
}
t.Run("wait for an element being visible", func(t *testing.T) {

2
v2/pkg/protocols/http/build_request.go
View File

@ -199,7 +199,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}
method, err := expressions.Evaluate(r.request.Method, finalValues)
method, err := expressions.Evaluate(r.request.Method.String(), finalValues)
if err != nil {
return nil, errors.Wrap(err, "could not evaluate helper expressions")
}

6
v2/pkg/protocols/http/build_request_test.go
View File

@ -71,7 +71,7 @@ func TestMakeRequestFromModal(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}/login.php"},
Method: "POST",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Body: "username=test&password=pass",
Headers: map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
@ -103,7 +103,7 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?query=example"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -211,7 +211,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,

4
v2/pkg/protocols/http/cluster_test.go
View File

@ -10,6 +10,6 @@ func TestCanCluster(t *testing.T) {
req := &Request{Unsafe: true}
require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request")
req = &Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}
require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}), "could not cluster GET request")
req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request")
}

6
v2/pkg/protocols/http/http.go
View File

@ -66,7 +66,7 @@ type Request struct {
// - "TRACE"
// - "PATCH"
// - "PURGE"
Method string `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"`
Method HTTPMethodTypeHolder `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"`
// description: |
// Body is an optional parameter which contains HTTP Request body.
// examples:
@ -248,7 +248,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
var hasPayloadName bool
// search for markers in all request parts
var inputs []string
inputs = append(inputs, request.Method, request.Body)
inputs = append(inputs, request.Method.String(), request.Body)
inputs = append(inputs, request.Raw...)
for k, v := range request.customHeaders {
inputs = append(inputs, fmt.Sprintf("%s: %s", k, v))
@ -258,7 +258,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
}
for _, input := range inputs {
if expressions.ContainsVariablesWithNames(input, map[string]interface{}{name: payload}) == nil {
if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input) == nil {
hasPayloadName = true
break
}

112
v2/pkg/protocols/http/http_method_types.go Normal file
View File

@ -0,0 +1,112 @@
package http
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// HTTPMethodType is the type of the method specified
type HTTPMethodType int
const (
HTTPGet HTTPMethodType = iota + 1
HTTPHead
HTTPPost
HTTPPut
HTTPDelete
HTTPConnect
HTTPOptions
HTTPTrace
HTTPPatch
HTTPPurge
//limit
limit
)
// HTTPMethodMapping is a table for conversion of method from string.
var HTTPMethodMapping = map[HTTPMethodType]string{
HTTPGet: "GET",
HTTPHead: "HEAD",
HTTPPost: "POST",
HTTPPut: "PUT",
HTTPDelete: "DELETE",
HTTPConnect: "CONNECT",
HTTPOptions: "OPTIONS",
HTTPTrace: "TRACE",
HTTPPatch: "PATCH",
HTTPPurge: "PURGE",
}
// GetSupportedHTTPMethodTypes returns list of supported types
func GetSupportedHTTPMethodTypes() []HTTPMethodType {
var result []HTTPMethodType
for index := HTTPMethodType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toHTTPMethodTypes(valueToMap string) (HTTPMethodType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range HTTPMethodMapping {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid HTTP method verb: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToUpper(value))
}
func (t HTTPMethodType) String() string {
return HTTPMethodMapping[t]
}
// HTTPMethodTypeHolder is used to hold internal type of the HTTP Method
type HTTPMethodTypeHolder struct {
MethodType HTTPMethodType
}
func (holder HTTPMethodTypeHolder) String() string {
return holder.MethodType.String()
}
func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "method is the HTTP request method",
Description: "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE",
}
for _, types := range GetSupportedHTTPMethodTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *HTTPMethodTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toHTTPMethodTypes(marshalledTypes)
if err != nil {
return err
}
holder.MethodType = computedType
return nil
}
func (holder *HTTPMethodTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.MethodType.String())
}
func (holder HTTPMethodTypeHolder) MarshalYAML() (interface{}, error) {
return holder.MethodType.String(), nil
}

22
v2/pkg/protocols/http/operators_test.go
View File

@ -25,7 +25,7 @@ func TestResponseToDSLMap(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -55,7 +55,7 @@ func TestHTTPOperatorMatch(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -143,7 +143,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
@ -166,7 +166,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -179,7 +179,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"test_header"},
}
err = extractor.CompileExtractors()
@ -195,7 +195,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("jq-simple", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".batters | .batter | .[] | .id"},
}
err = extractor.CompileExtractors()
@ -207,7 +207,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
})
t.Run("jq-array", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".array"},
}
err = extractor.CompileExtractors()
@ -219,7 +219,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
})
t.Run("jq-object", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".batters"},
}
err = extractor.CompileExtractors()
@ -235,7 +235,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
event["body"] = exampleResponseBody
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"TEST_HEADER"}, // only applies to KVal
CaseInsensitive: true,
}
@ -257,7 +257,7 @@ func TestHTTPMakeResult(t *testing.T) {
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
@ -267,7 +267,7 @@ func TestHTTPMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

8
v2/pkg/protocols/network/network.go
View File

@ -105,7 +105,7 @@ type Input struct {
// values:
// - "hex"
// - "text"
Type string `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"`
Type NetworkInputTypeHolder `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"`
// description: |
// Read is the number of bytes to read from socket.
//
@ -153,7 +153,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
}
// Pre-compile any input dsl functions before executing the request.
for _, input := range request.Inputs {
if input.Type != "" {
if input.Type.String() != "" {
continue
}
if compiled, evalErr := expressions.Evaluate(input.Data, map[string]interface{}{}); evalErr == nil {
@ -167,10 +167,10 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
// check if inputs contains the payload
var hasPayloadName bool
for _, input := range request.Inputs {
if input.Type != "" {
if input.Type.String() != "" {
continue
}
if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil {
if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input.Data) == nil {
hasPayloadName = true
break
}

100
v2/pkg/protocols/network/network_input_types.go Normal file
View File

@ -0,0 +1,100 @@
package network
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// NetworkInputType is the type of the method specified
type NetworkInputType int
const (
hexType NetworkInputType = iota + 1
textType
//limit
limit
)
// NetworkInputMapping is a table for conversion of method from string.
var NetworkInputMapping = map[NetworkInputType]string{
hexType: "hex",
textType: "text",
}
// GetSupportedNetworkInputTypes returns list of supported types
func GetSupportedNetworkInputTypes() []NetworkInputType {
var result []NetworkInputType
for index := NetworkInputType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toNetworkInputTypes(valueToMap string) (NetworkInputType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range NetworkInputMapping {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid network type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t NetworkInputType) String() string {
return NetworkInputMapping[t]
}
// NetworkInputTypeHolder is used to hold internal type of the Network type
type NetworkInputTypeHolder struct {
NetworkInputType NetworkInputType
}
func (holder NetworkInputTypeHolder) GetType() NetworkInputType {
return holder.NetworkInputType
}
func (holder NetworkInputTypeHolder) String() string {
return holder.NetworkInputType.String()
}
func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type is the type of input data",
Description: "description=Type of input specified in data field,enum=hex,enum=text",
}
for _, types := range GetSupportedNetworkInputTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *NetworkInputTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toNetworkInputTypes(marshalledTypes)
if err != nil {
return err
}
holder.NetworkInputType = computedType
return nil
}
func (holder *NetworkInputTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.NetworkInputType.String())
}
func (holder NetworkInputTypeHolder) MarshalYAML() (interface{}, error) {
return holder.NetworkInputType.String(), nil
}

6
v2/pkg/protocols/network/operators_test.go
View File

@ -149,7 +149,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -162,7 +162,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"request"},
}
err = extractor.CompileExtractors()
@ -193,7 +193,7 @@ func TestNetworkMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

4
v2/pkg/protocols/network/request.go
View File

@ -136,8 +136,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
for _, input := range request.Inputs {
var data []byte
switch input.Type {
case "hex":
switch input.Type.GetType() {
case hexType:
data, err = hex.DecodeString(input.Data)
default:
data = []byte(input.Data)

4
v2/pkg/protocols/network/request_test.go
View File

@ -38,7 +38,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"<h1>.*</h1>"},
}},
},
@ -91,7 +91,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
require.Equal(t, "<h1>Example Domain</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
request.Inputs[0].Type = "hex"
request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType}
request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)))
t.Run("hex-to-string", func(t *testing.T) {

6
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -139,7 +139,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -152,7 +152,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"test-header"},
Part: "header",
}
@ -184,7 +184,7 @@ func TestHTTPMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
}}

38
v2/pkg/templates/templates_doc.go
View File

@ -344,7 +344,7 @@ func init() {
"clusterbomb",
}
HTTPRequestDoc.Fields[8].Name = "method"
HTTPRequestDoc.Fields[8].Type = "string"
HTTPRequestDoc.Fields[8].Type = "HTTPMethodTypeHolder"
HTTPRequestDoc.Fields[8].Note = ""
HTTPRequestDoc.Fields[8].Description = "Method is the HTTP Request Method."
HTTPRequestDoc.Fields[8].Comments[encoder.LineComment] = "Method is the HTTP Request Method."
@ -649,7 +649,7 @@ func init() {
EXTRACTORSExtractorDoc.Fields[0].AddExample("", "cookie-extractor")
EXTRACTORSExtractorDoc.Fields[1].Name = "type"
EXTRACTORSExtractorDoc.Fields[1].Type = "string"
EXTRACTORSExtractorDoc.Fields[1].Type = "TypeHolder"
EXTRACTORSExtractorDoc.Fields[1].Note = ""
EXTRACTORSExtractorDoc.Fields[1].Description = "Type is the type of the extractor."
EXTRACTORSExtractorDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of the extractor."
@ -757,7 +757,7 @@ func init() {
FieldName: "dns",
},
}
DNSRequestDoc.Fields = make([]encoder.Doc, 10)
DNSRequestDoc.Fields = make([]encoder.Doc, 12)
DNSRequestDoc.Fields[0].Name = "matchers"
DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher"
DNSRequestDoc.Fields[0].Note = ""
@ -790,7 +790,7 @@ func init() {
DNSRequestDoc.Fields[4].AddExample("", "{{FQDN}}")
DNSRequestDoc.Fields[5].Name = "type"
DNSRequestDoc.Fields[5].Type = "string"
DNSRequestDoc.Fields[5].Type = "DNSRequestTypeHolder"
DNSRequestDoc.Fields[5].Note = ""
DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make."
DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make."
@ -825,16 +825,28 @@ func init() {
DNSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Retries is the number of retries for the DNS request"
DNSRequestDoc.Fields[7].AddExample("Use a retry of 3 to 5 generally", 5)
DNSRequestDoc.Fields[8].Name = "recursion"
DNSRequestDoc.Fields[8].Name = "trace"
DNSRequestDoc.Fields[8].Type = "bool"
DNSRequestDoc.Fields[8].Note = ""
DNSRequestDoc.Fields[8].Description = "Recursion determines if resolver should recurse all records to get fresh results."
DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results."
DNSRequestDoc.Fields[9].Name = "resolvers"
DNSRequestDoc.Fields[9].Type = "[]string"
DNSRequestDoc.Fields[8].Description = "Trace performs a trace operation for the target."
DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Trace performs a trace operation for the target."
DNSRequestDoc.Fields[9].Name = "trace-max-recursion"
DNSRequestDoc.Fields[9].Type = "int"
DNSRequestDoc.Fields[9].Note = ""
DNSRequestDoc.Fields[9].Description = "Resolvers to use for the dns requests"
DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = " Resolvers to use for the dns requests"
DNSRequestDoc.Fields[9].Description = "TraceMaxRecursion is the number of max recursion allowed for trace operations"
DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = "TraceMaxRecursion is the number of max recursion allowed for trace operations"
DNSRequestDoc.Fields[9].AddExample("Use a retry of 100 to 150 generally", 100)
DNSRequestDoc.Fields[10].Name = "recursion"
DNSRequestDoc.Fields[10].Type = "bool"
DNSRequestDoc.Fields[10].Note = ""
DNSRequestDoc.Fields[10].Description = "Recursion determines if resolver should recurse all records to get fresh results."
DNSRequestDoc.Fields[10].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results."
DNSRequestDoc.Fields[11].Name = "resolvers"
DNSRequestDoc.Fields[11].Type = "[]string"
DNSRequestDoc.Fields[11].Note = ""
DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests"
DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests"
FILERequestDoc.Type = "file.Request"
FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations."
@ -997,7 +1009,7 @@ func init() {
NETWORKInputDoc.Fields[0].AddExample("", "hex_decode('50494e47')")
NETWORKInputDoc.Fields[1].Name = "type"
NETWORKInputDoc.Fields[1].Type = "string"
NETWORKInputDoc.Fields[1].Type = "NetworkInputTypeHolder"
NETWORKInputDoc.Fields[1].Note = ""
NETWORKInputDoc.Fields[1].Description = "Type is the type of input specified in `data` field.\n\nDefault value is text, but hex can be used for hex formatted data."
NETWORKInputDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of input specified in `data` field."
@ -1086,7 +1098,7 @@ func init() {
ENGINEActionDoc.Fields[2].Description = "Description is the optional description of the headless action"
ENGINEActionDoc.Fields[2].Comments[encoder.LineComment] = "Description is the optional description of the headless action"
ENGINEActionDoc.Fields[3].Name = "action"
ENGINEActionDoc.Fields[3].Type = "string"
ENGINEActionDoc.Fields[3].Type = "ActionTypeHolder"
ENGINEActionDoc.Fields[3].Note = ""
ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform."
ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform."

8
v2/pkg/templates/templates_doc_examples.go
View File

@ -24,7 +24,7 @@ var (
Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"},
}
exampleNormalHTTPRequest = &http.Request{
Method: "GET",
Method: http.HTTPMethodTypeHolder{MethodType: http.HTTPGet},
Path: []string{"{{BaseURL}}/.git/config"},
Operators: operators.Operators{
MatchersCondition: "and",
@ -38,13 +38,13 @@ var (
exampleNormalDNSRequest = &dns.Request{
Name: "{{FQDN}}",
RequestType: "CNAME",
RequestType: dns.DNSRequestTypeHolder{DNSRequestType: dns.CNAME},
Class: "inet",
Retries: 2,
Recursion: true,
Operators: operators.Operators{
Extractors: []*extractors.Extractor{
{Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}},
{Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}},
},
},
}
@ -54,7 +54,7 @@ var (
Extensions: []string{"all"},
Operators: operators.Operators{
Extractors: []*extractors.Extractor{
{Type: "regex", Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}},
{Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}},
},
},
}