Merge branch 'dev' into cli-variables-as-payload

This commit is contained in:
Ice3man 2021-10-09 19:57:48 +05:30 committed by GitHub
commit e79c6262b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 254 additions and 100 deletions

View File

@ -71,6 +71,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
)

View File

@ -100,6 +100,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
github.com/projectdiscovery/mapcidr v0.0.8 // indirect
github.com/projectdiscovery/networkpolicy v0.0.1 // indirect

View File

@ -562,9 +562,9 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i
github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ=
github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I=
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs=
github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=

View File

@ -344,6 +344,7 @@ func (r *Runner) RunEnumeration() error {
IncludeTemplates: r.options.IncludeTemplates,
Authors: r.options.Author,
Severities: r.options.Severities,
ExcludeSeverities: r.options.ExcludeSeverities,
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Catalog: r.catalog,

View File

@ -11,6 +11,7 @@ import (
type TagFilter struct {
allowedTags map[string]struct{}
severities map[severity.Severity]struct{}
excludeSeverities map[severity.Severity]struct{}
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
@ -54,15 +55,21 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
}
func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined {
if (len(tagFilter.excludeSeverities) == 0 && len(tagFilter.severities) == 0) || templateSeverity == severity.Undefined {
return true
}
if _, ok := tagFilter.severities[templateSeverity]; ok {
return true
included := true
if len(tagFilter.severities) > 0 {
_, included = tagFilter.severities[templateSeverity]
}
return false
excluded := false
if len(tagFilter.excludeSeverities) > 0 {
_, excluded = tagFilter.excludeSeverities[templateSeverity]
}
return included && !excluded
}
func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
@ -114,17 +121,19 @@ type Config struct {
ExcludeTags []string
Authors []string
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
}
// New returns a tag filter for nuclei tag based execution
//
// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
// It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags.
func New(config *Config) *TagFilter {
filter := &TagFilter{
allowedTags: make(map[string]struct{}),
authors: make(map[string]struct{}),
severities: make(map[severity.Severity]struct{}),
excludeSeverities: make(map[severity.Severity]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
}
@ -140,6 +149,11 @@ func New(config *Config) *TagFilter {
filter.severities[tag] = struct{}{}
}
}
for _, tag := range config.ExcludeSeverities {
if _, ok := filter.excludeSeverities[tag]; !ok {
filter.excludeSeverities[tag] = struct{}{}
}
}
for _, tag := range config.Authors {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.authors[val]; !ok {

View File

@ -73,6 +73,16 @@ func TestTagBasedFilter(t *testing.T) {
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-severity", func(t *testing.T) {
filter := New(&Config{
ExcludeSeverities: severity.Severities{severity.Low},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
filter := New(&Config{
Tags: []string{"tag"},

View File

@ -23,6 +23,7 @@ type Config struct {
ExcludeTags []string
Authors []string
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
Catalog *catalog.Catalog
@ -53,6 +54,7 @@ func New(config *Config) (*Store, error) {
ExcludeTags: config.ExcludeTags,
Authors: config.Authors,
Severities: config.Severities,
ExcludeSeverities: config.ExcludeSeverities,
IncludeTags: config.IncludeTags,
}),
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{

View File

@ -18,6 +18,21 @@ var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`)
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func Evaluate(data string, base map[string]interface{}) (string, error) {
return evaluate(data, base)
}
// EvaluateByte checks if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
finalData, err := evaluate(string(data), base)
return []byte(finalData), err
}
func evaluate(data string, base map[string]interface{}) (string, error) {
data = replacer.Replace(data, base)
dynamicValues := make(map[string]interface{})
@ -37,30 +52,3 @@ func Evaluate(data string, base map[string]interface{}) (string, error) {
// Replacer dynamic values if any in raw request and parse it
return replacer.Replace(data, dynamicValues), nil
}
// EvaluateByte checks if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
final := replacer.Replace(string(data), base)
dynamicValues := make(map[string]interface{})
for _, match := range templateExpressionRegex.FindAllString(final, -1) {
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
if err != nil {
continue
}
result, err := compiled.Evaluate(base)
if err != nil {
continue
}
dynamicValues[expr] = result
}
// Replacer dynamic values if any in raw request and parse it
return []byte(replacer.Replace(final, dynamicValues)), nil
}

View File

@ -2,6 +2,8 @@
package generators
import "github.com/pkg/errors"
// Generator is the generator struct for generating payloads
type Generator struct {
Type Type
@ -12,8 +14,8 @@ type Generator struct {
type Type int
const (
// Sniper replaces each variable with values at a time.
Sniper Type = iota + 1
// Sniper replaces one iteration of the payload with a value.
BatteringRam Type = iota + 1
// PitchFork replaces variables with positional value from multiple wordlists
PitchFork
// ClusterBomb replaces variables with all possible combinations of values
@ -22,7 +24,7 @@ const (
// StringToType is a table for conversion of attack type from string.
var StringToType = map[string]Type{
"sniper": Sniper,
"batteringram": BatteringRam,
"pitchfork": PitchFork,
"clusterbomb": ClusterBomb,
}
@ -41,6 +43,12 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
generator.Type = payloadType
generator.payloads = compiled
// Validate the sniper/batteringram payload set
if payloadType == BatteringRam {
if len(payloads) != 1 {
return nil, errors.New("sniper/batteringram must have single payload set")
}
}
return generator, nil
}
@ -87,7 +95,7 @@ func (i *Iterator) Remaining() int {
func (i *Iterator) Total() int {
count := 0
switch i.Type {
case Sniper:
case BatteringRam:
for _, p := range i.payloads {
count += len(p.values)
}
@ -110,19 +118,19 @@ func (i *Iterator) Total() int {
// Value returns the next value for an iterator
func (i *Iterator) Value() (map[string]interface{}, bool) {
switch i.Type {
case Sniper:
return i.sniperValue()
case BatteringRam:
return i.batteringRamValue()
case PitchFork:
return i.pitchforkValue()
case ClusterBomb:
return i.clusterbombValue()
default:
return i.sniperValue()
return i.batteringRamValue()
}
}
// sniperValue returns a list of all payloads for the iterator
func (i *Iterator) sniperValue() (map[string]interface{}, bool) {
// batteringRamValue returns a list of all payloads for the iterator
func (i *Iterator) batteringRamValue() (map[string]interface{}, bool) {
values := make(map[string]interface{}, 1)
currentIndex := i.msbIterator
@ -132,7 +140,7 @@ func (i *Iterator) sniperValue() (map[string]interface{}, bool) {
if i.msbIterator == len(i.payloads) {
return nil, false
}
return i.sniperValue()
return i.batteringRamValue()
}
values[payload.name] = payload.value()
payload.incrementPosition()

View File

@ -6,11 +6,10 @@ import (
"github.com/stretchr/testify/require"
)
func TestSniperGenerator(t *testing.T) {
func TestBatteringRamGenerator(t *testing.T) {
usernames := []string{"admin", "password"}
moreUsernames := []string{"login", "test"}
generator, err := New(map[string]interface{}{"username": usernames, "aliases": moreUsernames}, Sniper, "")
generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "")
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -22,7 +21,7 @@ func TestSniperGenerator(t *testing.T) {
}
count++
}
require.Equal(t, len(usernames)+len(moreUsernames), count, "could not get correct sniper counts")
require.Equal(t, len(usernames), count, "could not get correct batteringram counts")
}
func TestPitchforkGenerator(t *testing.T) {

View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
)
var _ protocols.Request = &Request{}
@ -29,9 +30,14 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
return errors.Wrap(err, "could not build request")
}
requestString := compiledRequest.String()
if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr)
return nil
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain)
gologger.Print().Msgf("%s", compiledRequest.String())
gologger.Print().Msgf("%s", requestString)
}
// Send the request to the target servers

View File

@ -261,9 +261,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Payloads) > 0 {
attackType := r.AttackType
if attackType == "" {
attackType = "sniper"
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
r.attackType = generators.StringToType[attackType]
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {

View File

@ -1,6 +1,7 @@
package network
import (
"fmt"
"net"
"strings"
@ -176,9 +177,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Payloads) > 0 {
attackType := r.AttackType
if attackType == "" {
attackType = "sniper"
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
r.attackType = generators.StringToType[attackType]
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {

View File

@ -4,20 +4,41 @@ import (
"bufio"
"errors"
"net/http"
"regexp"
"strings"
)
var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `)
// readResponseFromString reads a raw http response from a string.
func readResponseFromString(data string) (*http.Response, error) {
var final string
if strings.HasPrefix(data, "HTTP/") {
final = data
final = addMinorVersionToHTTP(data)
} else {
lastIndex := strings.LastIndex(data, "HTTP/")
if lastIndex == -1 {
return nil, errors.New("malformed raw http response")
}
final = data[lastIndex:] // choose last http/ in case of it being later.
final = addMinorVersionToHTTP(final)
}
return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil)
}
// addMinorVersionToHTTP tries to add a minor version to http status header
// fixing the compatibility issue with go standard library.
func addMinorVersionToHTTP(data string) string {
matches := noMinor.FindAllStringSubmatch(data, 1)
if len(matches) == 0 {
return data
}
if len(matches[0]) < 2 {
return data
}
replacedVersion := strings.Replace(matches[0][0], matches[0][1], matches[0][1]+".0", 1)
data = strings.Replace(data, matches[0][0], replacedVersion, 1)
return data
}

View File

@ -2,7 +2,10 @@ package offlinehttp
import (
"io/ioutil"
"net/http"
"net/http/httputil"
"testing"
"time"
"github.com/stretchr/testify/require"
)
@ -18,11 +21,15 @@ func TestReadResponseFromString(t *testing.T) {
<h1>What is the Firing Range?</h1>
<p>
</body>
</body>
</html>`
t.Run("response", func(t *testing.T) {
data := `HTTP/1.1 200 OK
tests := []struct {
name string
data string
}{
{
name: "response",
data: `HTTP/1.1 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
@ -38,19 +45,51 @@ Server: Google Frontend
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "response-http2-without-minor-version",
data: `HTTP/2 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`
resp, err := readResponseFromString(data)
require.Nil(t, err, "could not read response from string")
</html>`,
},
{
name: "response-http2-with-minor-version",
data: `HTTP/2.0 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
respData, err := ioutil.ReadAll(resp.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
t.Run("request-response", func(t *testing.T) {
data := `GET http://public-firing-range.appspot.com/ HTTP/1.1
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "request-response",
data: `GET http://public-firing-range.appspot.com/ HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Upgrade-Insecure-Requests: 1
@ -72,9 +111,39 @@ Server: Google Frontend
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "request-response-without-minor-version",
data: `GET http://public-firing-range.appspot.com/ HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36
HTTP/2 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`
resp, err := readResponseFromString(data)
</html>`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := readResponseFromString(tt.data)
require.Nil(t, err, "could not read response from string")
respData, err := ioutil.ReadAll(resp.Body)
@ -83,3 +152,26 @@ Server: Google Frontend
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
}
t.Run("test-live-response-with-content-length", func(t *testing.T) {
client := &http.Client{
Timeout: 3 * time.Second,
}
data, err := client.Get("https://golang.org/doc/install")
require.Nil(t, err, "could not dial url")
defer data.Body.Close()
b, err := httputil.DumpResponse(data, true)
require.Nil(t, err, "could not dump response")
respData, err := readResponseFromString(string(b))
require.Nil(t, err, "could not read response from string")
_, err = ioutil.ReadAll(respData.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, "Google Frontend", respData.Header.Get("Server"), "could not get correct headers")
})
}

View File

@ -27,6 +27,8 @@ type Options struct {
varsPayload map[string]interface{}
// Severities filters templates based on their severity and only run the matching ones.
Severities severity.Severities
// ExcludeSeverities specifies severities to exclude
ExcludeSeverities severity.Severities
// Author filters templates based on their author and only run the matching ones.
Author goflags.NormalizedStringSlice
// IncludeTags includes specified tags to be run even while being in denylist