Merge branch 'dev' into cli-variables-as-payload

This commit is contained in:
Ice3man 2021-10-09 19:57:48 +05:30 committed by GitHub
commit e79c6262b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 254 additions and 100 deletions

View File

@ -71,6 +71,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"),
flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.VarP(&options.ExcludeSeverities, "exclude-impact", "exclude-severity", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"),
)

View File

@ -100,6 +100,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca // indirect
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
github.com/projectdiscovery/mapcidr v0.0.8 // indirect
github.com/projectdiscovery/networkpolicy v0.0.1 // indirect

View File

@ -562,9 +562,9 @@ github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/i
github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ=
github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges=
github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ=
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I=
github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA=
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs=
github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=

View File

@ -344,6 +344,7 @@ func (r *Runner) RunEnumeration() error {
IncludeTemplates: r.options.IncludeTemplates,
Authors: r.options.Author,
Severities: r.options.Severities,
ExcludeSeverities: r.options.ExcludeSeverities,
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Catalog: r.catalog,

View File

@ -9,11 +9,12 @@ import (
// TagFilter is used to filter nuclei templates for tag based execution
type TagFilter struct {
allowedTags map[string]struct{}
severities map[severity.Severity]struct{}
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
allowedTags map[string]struct{}
severities map[severity.Severity]struct{}
excludeSeverities map[severity.Severity]struct{}
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
}
// ErrExcluded is returned for excluded templates
@ -54,15 +55,21 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
}
func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined {
if (len(tagFilter.excludeSeverities) == 0 && len(tagFilter.severities) == 0) || templateSeverity == severity.Undefined {
return true
}
if _, ok := tagFilter.severities[templateSeverity]; ok {
return true
included := true
if len(tagFilter.severities) > 0 {
_, included = tagFilter.severities[templateSeverity]
}
return false
excluded := false
if len(tagFilter.excludeSeverities) > 0 {
_, excluded = tagFilter.excludeSeverities[templateSeverity]
}
return included && !excluded
}
func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
@ -110,23 +117,25 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
}
type Config struct {
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
IncludeTags []string
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
}
// New returns a tag filter for nuclei tag based execution
//
// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
// It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags.
func New(config *Config) *TagFilter {
filter := &TagFilter{
allowedTags: make(map[string]struct{}),
authors: make(map[string]struct{}),
severities: make(map[severity.Severity]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
allowedTags: make(map[string]struct{}),
authors: make(map[string]struct{}),
severities: make(map[severity.Severity]struct{}),
excludeSeverities: make(map[severity.Severity]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
}
for _, tag := range config.ExcludeTags {
for _, val := range splitCommaTrim(tag) {
@ -140,6 +149,11 @@ func New(config *Config) *TagFilter {
filter.severities[tag] = struct{}{}
}
}
for _, tag := range config.ExcludeSeverities {
if _, ok := filter.excludeSeverities[tag]; !ok {
filter.excludeSeverities[tag] = struct{}{}
}
}
for _, tag := range config.Authors {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.authors[val]; !ok {

View File

@ -73,6 +73,16 @@ func TestTagBasedFilter(t *testing.T) {
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-severity", func(t *testing.T) {
filter := New(&Config{
ExcludeSeverities: severity.Severities{severity.Low},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
filter := New(&Config{
Tags: []string{"tag"},

View File

@ -19,11 +19,12 @@ type Config struct {
ExcludeTemplates []string
IncludeTemplates []string
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
IncludeTags []string
Tags []string
ExcludeTags []string
Authors []string
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
Catalog *catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
@ -49,11 +50,12 @@ func New(config *Config) (*Store, error) {
store := &Store{
config: config,
tagFilter: filter.New(&filter.Config{
Tags: config.Tags,
ExcludeTags: config.ExcludeTags,
Authors: config.Authors,
Severities: config.Severities,
IncludeTags: config.IncludeTags,
Tags: config.Tags,
ExcludeTags: config.ExcludeTags,
Authors: config.Authors,
Severities: config.Severities,
ExcludeSeverities: config.ExcludeSeverities,
IncludeTags: config.IncludeTags,
}),
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
IncludedTemplates: config.IncludeTemplates,

View File

@ -18,6 +18,21 @@ var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`)
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func Evaluate(data string, base map[string]interface{}) (string, error) {
return evaluate(data, base)
}
// EvaluateByte checks if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
finalData, err := evaluate(string(data), base)
return []byte(finalData), err
}
func evaluate(data string, base map[string]interface{}) (string, error) {
data = replacer.Replace(data, base)
dynamicValues := make(map[string]interface{})
@ -37,30 +52,3 @@ func Evaluate(data string, base map[string]interface{}) (string, error) {
// Replacer dynamic values if any in raw request and parse it
return replacer.Replace(data, dynamicValues), nil
}
// EvaluateByte checks if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {
final := replacer.Replace(string(data), base)
dynamicValues := make(map[string]interface{})
for _, match := range templateExpressionRegex.FindAllString(final, -1) {
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
if err != nil {
continue
}
result, err := compiled.Evaluate(base)
if err != nil {
continue
}
dynamicValues[expr] = result
}
// Replacer dynamic values if any in raw request and parse it
return []byte(replacer.Replace(final, dynamicValues)), nil
}

View File

@ -2,6 +2,8 @@
package generators
import "github.com/pkg/errors"
// Generator is the generator struct for generating payloads
type Generator struct {
Type Type
@ -12,8 +14,8 @@ type Generator struct {
type Type int
const (
// Sniper replaces each variable with values at a time.
Sniper Type = iota + 1
// Sniper replaces one iteration of the payload with a value.
BatteringRam Type = iota + 1
// PitchFork replaces variables with positional value from multiple wordlists
PitchFork
// ClusterBomb replaces variables with all possible combinations of values
@ -22,9 +24,9 @@ const (
// StringToType is a table for conversion of attack type from string.
var StringToType = map[string]Type{
"sniper": Sniper,
"pitchfork": PitchFork,
"clusterbomb": ClusterBomb,
"batteringram": BatteringRam,
"pitchfork": PitchFork,
"clusterbomb": ClusterBomb,
}
// New creates a new generator structure for payload generation
@ -41,6 +43,12 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string)
generator.Type = payloadType
generator.payloads = compiled
// Validate the sniper/batteringram payload set
if payloadType == BatteringRam {
if len(payloads) != 1 {
return nil, errors.New("sniper/batteringram must have single payload set")
}
}
return generator, nil
}
@ -87,7 +95,7 @@ func (i *Iterator) Remaining() int {
func (i *Iterator) Total() int {
count := 0
switch i.Type {
case Sniper:
case BatteringRam:
for _, p := range i.payloads {
count += len(p.values)
}
@ -110,19 +118,19 @@ func (i *Iterator) Total() int {
// Value returns the next value for an iterator
func (i *Iterator) Value() (map[string]interface{}, bool) {
switch i.Type {
case Sniper:
return i.sniperValue()
case BatteringRam:
return i.batteringRamValue()
case PitchFork:
return i.pitchforkValue()
case ClusterBomb:
return i.clusterbombValue()
default:
return i.sniperValue()
return i.batteringRamValue()
}
}
// sniperValue returns a list of all payloads for the iterator
func (i *Iterator) sniperValue() (map[string]interface{}, bool) {
// batteringRamValue returns a list of all payloads for the iterator
func (i *Iterator) batteringRamValue() (map[string]interface{}, bool) {
values := make(map[string]interface{}, 1)
currentIndex := i.msbIterator
@ -132,7 +140,7 @@ func (i *Iterator) sniperValue() (map[string]interface{}, bool) {
if i.msbIterator == len(i.payloads) {
return nil, false
}
return i.sniperValue()
return i.batteringRamValue()
}
values[payload.name] = payload.value()
payload.incrementPosition()

View File

@ -6,11 +6,10 @@ import (
"github.com/stretchr/testify/require"
)
func TestSniperGenerator(t *testing.T) {
func TestBatteringRamGenerator(t *testing.T) {
usernames := []string{"admin", "password"}
moreUsernames := []string{"login", "test"}
generator, err := New(map[string]interface{}{"username": usernames, "aliases": moreUsernames}, Sniper, "")
generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "")
require.Nil(t, err, "could not create generator")
iterator := generator.NewIterator()
@ -22,7 +21,7 @@ func TestSniperGenerator(t *testing.T) {
}
count++
}
require.Equal(t, len(usernames)+len(moreUsernames), count, "could not get correct sniper counts")
require.Equal(t, len(usernames), count, "could not get correct batteringram counts")
}
func TestPitchforkGenerator(t *testing.T) {

View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
)
var _ protocols.Request = &Request{}
@ -29,9 +30,14 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
return errors.Wrap(err, "could not build request")
}
requestString := compiledRequest.String()
if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr)
return nil
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain)
gologger.Print().Msgf("%s", compiledRequest.String())
gologger.Print().Msgf("%s", requestString)
}
// Send the request to the target servers

View File

@ -261,9 +261,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Payloads) > 0 {
attackType := r.AttackType
if attackType == "" {
attackType = "sniper"
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
r.attackType = generators.StringToType[attackType]
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {

View File

@ -1,6 +1,7 @@
package network
import (
"fmt"
"net"
"strings"
@ -176,9 +177,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Payloads) > 0 {
attackType := r.AttackType
if attackType == "" {
attackType = "sniper"
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
r.attackType = generators.StringToType[attackType]
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {

View File

@ -4,20 +4,41 @@ import (
"bufio"
"errors"
"net/http"
"regexp"
"strings"
)
var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `)
// readResponseFromString reads a raw http response from a string.
func readResponseFromString(data string) (*http.Response, error) {
var final string
if strings.HasPrefix(data, "HTTP/") {
final = data
final = addMinorVersionToHTTP(data)
} else {
lastIndex := strings.LastIndex(data, "HTTP/")
if lastIndex == -1 {
return nil, errors.New("malformed raw http response")
}
final = data[lastIndex:] // choose last http/ in case of it being later.
final = addMinorVersionToHTTP(final)
}
return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil)
}
// addMinorVersionToHTTP tries to add a minor version to http status header
// fixing the compatibility issue with go standard library.
func addMinorVersionToHTTP(data string) string {
matches := noMinor.FindAllStringSubmatch(data, 1)
if len(matches) == 0 {
return data
}
if len(matches[0]) < 2 {
return data
}
replacedVersion := strings.Replace(matches[0][0], matches[0][1], matches[0][1]+".0", 1)
data = strings.Replace(data, matches[0][0], replacedVersion, 1)
return data
}

View File

@ -2,7 +2,10 @@ package offlinehttp
import (
"io/ioutil"
"net/http"
"net/http/httputil"
"testing"
"time"
"github.com/stretchr/testify/require"
)
@ -18,11 +21,15 @@ func TestReadResponseFromString(t *testing.T) {
<h1>What is the Firing Range?</h1>
<p>
</body>
</body>
</html>`
t.Run("response", func(t *testing.T) {
data := `HTTP/1.1 200 OK
tests := []struct {
name string
data string
}{
{
name: "response",
data: `HTTP/1.1 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
@ -38,19 +45,51 @@ Server: Google Frontend
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "response-http2-without-minor-version",
data: `HTTP/2 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`
resp, err := readResponseFromString(data)
require.Nil(t, err, "could not read response from string")
</html>`,
},
{
name: "response-http2-with-minor-version",
data: `HTTP/2.0 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
respData, err := ioutil.ReadAll(resp.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
t.Run("request-response", func(t *testing.T) {
data := `GET http://public-firing-range.appspot.com/ HTTP/1.1
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "request-response",
data: `GET http://public-firing-range.appspot.com/ HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Upgrade-Insecure-Requests: 1
@ -72,14 +111,67 @@ Server: Google Frontend
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`,
},
{
name: "request-response-without-minor-version",
data: `GET http://public-firing-range.appspot.com/ HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36
HTTP/2 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</html>`
resp, err := readResponseFromString(data)
</html>`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := readResponseFromString(tt.data)
require.Nil(t, err, "could not read response from string")
respData, err := ioutil.ReadAll(resp.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
}
t.Run("test-live-response-with-content-length", func(t *testing.T) {
client := &http.Client{
Timeout: 3 * time.Second,
}
data, err := client.Get("https://golang.org/doc/install")
require.Nil(t, err, "could not dial url")
defer data.Body.Close()
b, err := httputil.DumpResponse(data, true)
require.Nil(t, err, "could not dump response")
respData, err := readResponseFromString(string(b))
require.Nil(t, err, "could not read response from string")
respData, err := ioutil.ReadAll(resp.Body)
_, err = ioutil.ReadAll(respData.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
require.Equal(t, "Google Frontend", respData.Header.Get("Server"), "could not get correct headers")
})
}

View File

@ -27,6 +27,8 @@ type Options struct {
varsPayload map[string]interface{}
// Severities filters templates based on their severity and only run the matching ones.
Severities severity.Severities
// ExcludeSeverities specifies severities to exclude
ExcludeSeverities severity.Severities
// Author filters templates based on their author and only run the matching ones.
Author goflags.NormalizedStringSlice
// IncludeTags includes specified tags to be run even while being in denylist