External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 20:05:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added integration tests for variables + misc changes

Browse Source
This commit is contained in:
Ice3man 2022-04-02 02:14:00 +05:30
parent 17fa23e737
commit d0d65f8d6b
14 changed files with 211 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
integration_tests/dns/variables.yaml Normal file
View File

@ -0,0 +1,21 @@
id: variables-example
info:
name: Variables Example
author: pdteam
severity: info
variables:
a1: "FQDN"
a2: "IN"
dns:
- name: "{{a1}}"
type: A
class: inet
recursion: true
retries: 3
matchers:
- type: word
words:
- "{{a2}}"

20
integration_tests/headless/variables.yaml Normal file
View File

@ -0,0 +1,20 @@
id: variables-example
info:
name: Variables Example
author: pdteam
severity: info
variables:
a1: "value"
headless:
- steps:
- args:
url: "{{BaseURL}}"
action: navigate
- action: waitload
matchers:
- type: word
words:
- "{{a1}}"

26
integration_tests/http/variables.yaml Normal file
View File

@ -0,0 +1,26 @@
id: variables-example
info:
name: Variables Example
author: pdteam
severity: info
variables:
a1: "value"
a2: "rand_base(5)"
requests:
- raw:
- |
GET / HTTP/1.1
Host: {{FQDN}}
Test: {{a1}}
Another: {{a2}}
stop-at-first-match: true
matchers-condition: or
matchers:
- type: word
words:
- "{{a1}}"
- "{{a2}}"

23
integration_tests/network/variables.yaml Normal file
View File

@ -0,0 +1,23 @@
id: variables-example
info:
name: Variables Example
author: pdteam
severity: info
variables:
a1: "Hostname"
a2: "PING"
a3: "PONG"
network:
- host:
- "{{a1}}"
inputs:
- data: "{{a2}}\r\n"
read-size: 4
matchers:
- type: word
part: data
words:
- "{{a3}}"

23
v2/cmd/integration-test/dns.go
View File

@ -5,9 +5,10 @@ import (
) )
var dnsTestCases = map[string]testutils.TestCase{ var dnsTestCases = map[string]testutils.TestCase{
"dns/basic.yaml": &dnsBasic{}, "dns/basic.yaml": &dnsBasic{},
"dns/ptr.yaml": &dnsPtr{}, "dns/ptr.yaml": &dnsPtr{},
"dns/caa.yaml": &dnsCAA{}, "dns/caa.yaml": &dnsCAA{},
"dns/variables.yaml": &dnsVariables{},
} }
type dnsBasic struct{} type dnsBasic struct{}
@ -57,3 +58,19 @@ func (h *dnsCAA) Execute(filePath string) error {
} }
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type dnsVariables struct{}
// Execute executes a test case and returns an error if occurred
func (h *dnsVariables) Execute(filePath string) error {
var routerErr error
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug)
if err != nil {
return err
}
if routerErr != nil {
return routerErr
}
return expectResultsCount(results, 1)
}

19
v2/cmd/integration-test/headless.go
View File

@ -14,6 +14,7 @@ var headlessTestcases = map[string]testutils.TestCase{
"headless/headless-header-action.yaml": &headlessHeaderActions{}, "headless/headless-header-action.yaml": &headlessHeaderActions{},
"headless/headless-extract-values.yaml": &headlessExtractValues{}, "headless/headless-extract-values.yaml": &headlessExtractValues{},
"headless/headless-payloads.yaml": &headlessPayloads{}, "headless/headless-payloads.yaml": &headlessPayloads{},
"headless/variables.yaml": &headlessVariables{},
} }
type headlessBasic struct{} type headlessBasic struct{}
@ -92,3 +93,21 @@ func (h *headlessPayloads) Execute(filePath string) error {
return expectResultsCount(results, 4) return expectResultsCount(results, 4)
} }
type headlessVariables struct{}
// Execute executes a test case and returns an error if occurred
func (h *headlessVariables) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
_, _ = w.Write([]byte("<html><body>value</body></html>"))
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}

32
v2/cmd/integration-test/network.go
View File

@ -11,6 +11,7 @@ var networkTestcases = map[string]testutils.TestCase{
"network/hex.yaml": &networkBasic{}, "network/hex.yaml": &networkBasic{},
"network/multi-step.yaml": &networkMultiStep{}, "network/multi-step.yaml": &networkMultiStep{},
"network/self-contained.yaml": &networkRequestSelContained{}, "network/self-contained.yaml": &networkRequestSelContained{},
"network/variables.yaml": &networkVariables{},
} }
const defaultStaticPort = 5431 const defaultStaticPort = 5431
@ -116,3 +117,34 @@ func (h *networkRequestSelContained) Execute(filePath string) error {
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type networkVariables struct{}
// Execute executes a test case and returns an error if occurred
func (h *networkVariables) Execute(filePath string) error {
var routerErr error
ts := testutils.NewTCPServer(nil, defaultStaticPort, func(conn net.Conn) {
defer conn.Close()
data := make([]byte, 4)
if _, err := conn.Read(data); err != nil {
routerErr = err
return
}
if string(data) == "PING" {
_, _ = conn.Write([]byte("PONG"))
}
})
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
if routerErr != nil {
return routerErr
}
return expectResultsCount(results, 1)
}

24
v2/pkg/protocols/dns/dns.go
View File

@ -8,11 +8,9 @@ import (
"github.com/weppos/publicsuffix-go/publicsuffix" "github.com/weppos/publicsuffix-go/publicsuffix"
"github.com/projectdiscovery/iputil"
"github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/retryabledns" "github.com/projectdiscovery/retryabledns"
@ -171,33 +169,13 @@ func (request *Request) Requests() int {
} }
// Make returns the request to be sent for the protocol // Make returns the request to be sent for the protocol
func (request *Request) Make(host string) (*dns.Msg, error) { func (request *Request) Make(host string, vars map[string]interface{}) (*dns.Msg, error) {
isIP := iputil.IsIP(host)
switch {
case request.question == dns.TypePTR && isIP:
var err error
host, err = dns.ReverseAddr(host)
if err != nil {
return nil, err
}
default:
if isIP {
return nil, errors.New("cannot use IP address as DNS input")
}
host = dns.Fqdn(host)
}
// Build a request on the specified URL // Build a request on the specified URL
req := new(dns.Msg) req := new(dns.Msg)
req.Id = dns.Id() req.Id = dns.Id()
req.RecursionDesired = *request.Recursion req.RecursionDesired = *request.Recursion
var q dns.Question var q dns.Question
vars := GenerateDNSVariables(host)
variablesMap := request.options.Variables.Evaluate(vars)
vars = generators.MergeMaps(variablesMap, vars)
final := replacer.Replace(request.Name, vars) final := replacer.Replace(request.Name, vars)
q.Name = dns.Fqdn(final) q.Name = dns.Fqdn(final)

2
v2/pkg/protocols/dns/dns_test.go
View File

@ -42,7 +42,7 @@ func TestDNSCompileMake(t *testing.T) {
err := request.Compile(executerOpts) err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request") require.Nil(t, err, "could not compile dns request")
req, err := request.Make("one.one.one.one") req, err := request.Make("one.one.one.one", map[string]interface{}{})
require.Nil(t, err, "could not make dns request") require.Nil(t, err, "could not make dns request")
require.Equal(t, "one.one.one.one.", req.Question[0].Name, "could not get correct dns question") require.Equal(t, "one.one.one.one.", req.Question[0].Name, "could not get correct dns question")
} }

2
v2/pkg/protocols/dns/operators.go
View File

@ -34,7 +34,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
case matchers.SizeMatcher: case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{}
case matchers.WordsMatcher: case matchers.WordsMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), data))
case matchers.RegexMatcher: case matchers.RegexMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item)))
case matchers.BinaryMatcher: case matchers.BinaryMatcher:

36
v2/pkg/protocols/dns/request.go
View File

@ -4,12 +4,15 @@ import (
"encoding/hex" "encoding/hex"
"net/url" "net/url"
"github.com/miekg/dns"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/iputil"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -34,8 +37,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
domain = input domain = input
} }
var err error
domain, err = request.parseDNSInput(domain)
if err != nil {
return errors.Wrap(err, "could not build request")
}
vars := GenerateDNSVariables(domain)
variablesMap := request.options.Variables.Evaluate(vars)
vars = generators.MergeMaps(variablesMap, vars)
// Compile each request for the template based on the URL // Compile each request for the template based on the URL
compiledRequest, err := request.Make(domain) compiledRequest, err := request.Make(domain, vars)
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
@ -87,7 +99,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
for k, v := range previous { for k, v := range previous {
outputEvent[k] = v outputEvent[k] = v
} }
for k, v := range vars {
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
// TODO: dynamic values are not supported yet // TODO: dynamic values are not supported yet
@ -100,6 +114,24 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return nil return nil
} }
func (request *Request) parseDNSInput(host string) (string, error) {
isIP := iputil.IsIP(host)
switch {
case request.question == dns.TypePTR && isIP:
var err error
host, err = dns.ReverseAddr(host)
if err != nil {
return "", err
}
default:
if isIP {
return "", errors.New("cannot use IP address as DNS input")
}
host = dns.Fqdn(host)
}
return host, nil
}
func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) { func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) {
cliOptions := requestOptions.Options cliOptions := requestOptions.Options
if cliOptions.Debug || cliOptions.DebugResponse { if cliOptions.Debug || cliOptions.DebugResponse {

2
v2/pkg/protocols/headless/operators.go
View File

@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
case matchers.SizeMatcher: case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher: case matchers.WordsMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data))
case matchers.RegexMatcher: case matchers.RegexMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher: case matchers.BinaryMatcher:

3
v2/pkg/protocols/headless/request.go
View File

@ -106,6 +106,9 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
for k, v := range out { for k, v := range out {
outputEvent[k] = v outputEvent[k] = v
} }
for k, v := range payloads {
outputEvent[k] = v
}
var event *output.InternalWrappedEvent var event *output.InternalWrappedEvent
if len(page.InteractshURLs) == 0 { if len(page.InteractshURLs) == 0 {

17
v2/pkg/protocols/network/request.go
View File

@ -50,6 +50,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
for _, kv := range request.addresses { for _, kv := range request.addresses {
variables := generateNetworkVariables(address) variables := generateNetworkVariables(address)
variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, variables))
variables = generators.MergeMaps(variablesMap, variables)
actualAddress := replacer.Replace(kv.address, variables) actualAddress := replacer.Replace(kv.address, variables)
if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil {
@ -62,6 +64,9 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
// executeAddress executes the request for an address // executeAddress executes the request for an address
func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address})
payloads := generators.BuildPayloadFromOptions(request.options.Options)
if !strings.Contains(actualAddress, ":") { if !strings.Contains(actualAddress, ":") {
err := errors.New("no port provided in network protocol request") err := errors.New("no port provided in network protocol request")
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
@ -69,9 +74,6 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
return err return err
} }
variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address})
payloads := generators.BuildPayloadFromOptions(request.options.Options)
if request.generator != nil { if request.generator != nil {
iterator := request.generator.NewIterator() iterator := request.generator.NewIterator()
@ -101,9 +103,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
err error err error
) )
variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(variables, payloads))
payloads = generators.MergeMaps(variablesMap, payloads)
if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil {
hostname = host hostname = host
} }
@ -126,6 +125,8 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
responseBuilder := &strings.Builder{} responseBuilder := &strings.Builder{}
reqBuilder := &strings.Builder{} reqBuilder := &strings.Builder{}
interimValues := generators.MergeMaps(variables, payloads)
inputEvents := make(map[string]interface{}) inputEvents := make(map[string]interface{})
for _, input := range request.Inputs { for _, input := range request.Inputs {
var data []byte var data []byte
@ -149,7 +150,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
data = []byte(transformedData) data = []byte(transformedData)
} }
finalData, dataErr := expressions.EvaluateByte(data, payloads) finalData, dataErr := expressions.EvaluateByte(data, interimValues)
if dataErr != nil { if dataErr != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
@ -255,7 +256,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
for k, v := range previous { for k, v := range previous {
outputEvent[k] = v outputEvent[k] = v
} }
for k, v := range payloads { for k, v := range interimValues {
outputEvent[k] = v outputEvent[k] = v
} }
for k, v := range inputEvents { for k, v := range inputEvents {