External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 21:25:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[feature] Add coloring to debug information #999 [WIP]

Browse Source 
TODO:
* if there are multiple matchers, make sure the response is only displayed once, with all the matching values colored
* remove code duplication from the request.go files
This commit is contained in:
forgedhallpass 2021-09-29 19:43:46 +03:00
parent 8a8d61996f
commit 4be6b3cc96
22 changed files with 401 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
v2/pkg/operators/matchers/match.go
View File

@ -40,7 +40,8 @@ func (m *Matcher) MatchSize(length int) bool {
}
// MatchWords matches a word check against a corpus.
func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool {
func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) {
var matchedWords []string
// Iterate over all the words accepted as valid
for i, word := range m.Words {
if dynamicValues == nil {
@ -57,7 +58,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
@ -65,19 +66,21 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, []string{word}
}
matchedWords = append(matchedWords, word)
// If we are at the end of the words, return with true
if len(m.Words)-1 == i {
return true
return true, matchedWords
}
}
return false
return false, []string{}
}
// MatchRegex matches a regex check against a corpus
func (m *Matcher) MatchRegex(corpus string) bool {
func (m *Matcher) MatchRegex(corpus string) (bool, []string) {
// Iterate over all the regexes accepted as valid
for i, regex := range m.regexCompiled {
// Continue if the regex doesn't match
@ -85,7 +88,7 @@ func (m *Matcher) MatchRegex(corpus string) bool {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
@ -93,19 +96,19 @@ func (m *Matcher) MatchRegex(corpus string) bool {
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, regex.FindAllString(corpus, -1)
}
// If we are at the end of the regex, return with true
if len(m.regexCompiled)-1 == i {
return true
return true, []string{corpus}
}
}
return false
return false, []string{}
}
// MatchBinary matches a binary check against a corpus
func (m *Matcher) MatchBinary(corpus string) bool {
func (m *Matcher) MatchBinary(corpus string) (bool, []string) {
// Iterate over all the words accepted as valid
for i, binary := range m.Binary {
// Continue if the word doesn't match
@ -114,7 +117,7 @@ func (m *Matcher) MatchBinary(corpus string) bool {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
@ -122,15 +125,15 @@ func (m *Matcher) MatchBinary(corpus string) bool {
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, []string{string(hexa)}
}
// If we are at the end of the words, return with true
if len(m.Binary)-1 == i {
return true
return true, []string{string(hexa)}
}
}
return false
return false, []string{}
}
// MatchDSL matches on a generic map result

30
v2/pkg/operators/matchers/match_test.go
View File

@ -9,24 +9,29 @@ import (
func TestANDCondition(t *testing.T) {
m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
matched := m.MatchWords("a b", nil)
require.True(t, matched, "Could not match valid AND condition")
isMatched, matched := m.MatchWords("a b", nil)
require.True(t, isMatched, "Could not match valid AND condition")
require.Equal(t, m.Words, matched)
matched = m.MatchWords("b", nil)
require.False(t, matched, "Could match invalid AND condition")
isMatched, matched = m.MatchWords("b", nil)
require.False(t, isMatched, "Could match invalid AND condition")
require.Equal(t, []string{}, matched)
}
func TestORCondition(t *testing.T) {
m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}}
matched := m.MatchWords("a b", nil)
require.True(t, matched, "Could not match valid OR condition")
isMatched, matched := m.MatchWords("a b", nil)
require.True(t, isMatched, "Could not match valid OR condition")
require.Equal(t, []string{"a"}, matched)
matched = m.MatchWords("b", nil)
require.True(t, matched, "Could not match valid OR condition")
isMatched, matched = m.MatchWords("b", nil)
require.True(t, isMatched, "Could not match valid OR condition")
require.Equal(t, []string{"b"}, matched)
matched = m.MatchWords("c", nil)
require.False(t, matched, "Could match invalid OR condition")
isMatched, matched = m.MatchWords("c", nil)
require.False(t, isMatched, "Could match invalid OR condition")
require.Equal(t, []string{}, matched)
}
func TestHexEncoding(t *testing.T) {
@ -34,6 +39,7 @@ func TestHexEncoding(t *testing.T) {
err := m.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := m.MatchWords("PING", nil)
require.True(t, matched, "Could not match valid Hex condition")
isMatched, matched := m.MatchWords("PING", nil)
require.True(t, isMatched, "Could not match valid Hex condition")
require.Equal(t, m.Words, matched)
}

8
v2/pkg/operators/matchers/matchers.go
View File

@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool {
return data
}
// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string
func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) {
if m.Negative {
return !data, []string{}
}
return data, matchedSnippet
}
// GetType returns the type of the matcher
func (m *Matcher) GetType() MatcherType {
return m.matcherType

20
v2/pkg/operators/operators.go
View File

@ -100,7 +100,7 @@ func (r *Result) Merge(result *Result) {
}
// MatchFunc performs matching operation for a matcher on model and returns true or false.
type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string)
// ExtractFunc performs extracting operation for an extractor on model and returns true or false.
type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
@ -138,21 +138,19 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
for _, matcher := range r.Matchers {
// Check if the matcher matched
if !match(data, matcher) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
if len(result.DynamicValues) > 0 {
return result, true
}
return nil, false
}
} else {
if isMatch, _ := match(data, matcher); isMatch {
// If the matcher has matched, and it's an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && matcher.Name != "" {
result.Matches[matcher.Name] = struct{}{}
}
matches = true
} else if matcherCondition == matchers.ANDCondition {
if len(result.DynamicValues) > 0 {
return result, true
}
return nil, false
}
}
@ -162,7 +160,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
return result, true
}
// Don't print if we have matchers and they have not matched, regardless of extractor
// Don't print if we have matchers, and they have not matched, regardless of extractor
if len(r.Matchers) > 0 && !matches {
return nil, false
}

20
v2/pkg/protocols/dns/operators.go
View File

@ -14,7 +14,7 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "":
@ -23,24 +23,24 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
return matcher.Result(matcher.MatchStatusCode(item.(int)))
case matchers.StatusMatcher: // TODO is this correct?
return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(types.ToString(item))))
return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(types.ToString(item), nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(types.ToString(item)))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item)))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(types.ToString(item)))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item)))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

20
v2/pkg/protocols/dns/operators_test.go
View File

@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatch, matched := request.Match(event, matcher)
require.True(t, isMatch, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("rcode", func(t *testing.T) {
@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile rcode matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid rcode response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid rcode response")
require.Equal(t, []string{}, matched)
})
t.Run("negative", func(t *testing.T) {
@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}

58
v2/pkg/protocols/dns/request.go
View File

@ -2,9 +2,13 @@ package dns
import (
"net/url"
"strings"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
@ -48,27 +52,57 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
r.options.Output.Request(r.options.TemplateID, domain, "dns", err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain)
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain)
gologger.Print().Msgf("%s", resp.String())
}
outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
}
event := createEvent(r, domain, resp.String(), outputEvent)
callback(event)
return nil
}
// TODO extract duplicated code
func createEvent(request *Request, domain string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
debugResponse := func(data string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
gologger.Print().Msgf("%s", data)
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
isMatch, matched := request.Match(data, matcher)
var result string = response
if len(matched) != 0 {
if !request.options.Options.NoColor {
colorizer := aurora.NewAurora(true)
for _, currentMatch := range matched {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
debugResponse(result)
}
return isMatch, matched
}
result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
} else {
debugResponse(response)
}
return event
}
// isURL tests a string to determine if it is a well-structured url or not.
func isURL(toTest string) bool {
if _, err := url.ParseRequestURI(toTest); err != nil {

16
v2/pkg/protocols/file/operators.go
View File

@ -13,7 +13,7 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "data", "":
@ -22,23 +22,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

15
v2/pkg/protocols/file/operators_test.go
View File

@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}

76
v2/pkg/protocols/file/request.go
View File

@ -3,13 +3,17 @@ package file
import (
"io/ioutil"
"os"
"strings"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/remeh/sizedwaitgroup"
)
var _ protocols.Request = &Request{}
@ -21,50 +25,40 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
err := r.getInputPaths(input, func(data string) {
wg.Add()
go func(data string) {
go func(filePath string) {
defer wg.Done()
file, err := os.Open(data)
file, err := os.Open(filePath)
if err != nil {
gologger.Error().Msgf("Could not open file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err)
return
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err)
return
}
if stat.Size() >= int64(r.MaxSize) {
gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data)
gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath)
return
}
buffer, err := ioutil.ReadAll(file)
if err != nil {
gologger.Error().Msgf("Could not read file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err)
return
}
dataStr := tostring.UnsafeToString(buffer)
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data)
gologger.Print().Msgf("%s", dataStr)
}
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data)
outputEvent := r.responseToDSLMap(dataStr, input, data)
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, filePath)
outputEvent := r.responseToDSLMap(dataStr, input, filePath)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
}
event := createEvent(r, filePath, dataStr, outputEvent)
callback(event)
}(data)
})
@ -77,3 +71,43 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
r.options.Progress.IncrementRequests()
return nil
}
// TODO extract duplicated code
func createEvent(request *Request, filePath string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
debugResponse := func(data string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
gologger.Print().Msgf("%s", data)
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
isMatch, matched := request.Match(data, matcher)
var result = response
if len(matched) != 0 {
if !request.options.Options.NoColor {
colorizer := aurora.NewAurora(true)
for _, currentMatch := range matched {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
debugResponse(result)
}
return isMatch, matched
}
result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
} else {
debugResponse(response)
}
return event
}

16
v2/pkg/protocols/headless/operators.go
View File

@ -11,7 +11,7 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "resp", "":
@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

56
v2/pkg/protocols/headless/request.go
View File

@ -5,8 +5,11 @@ import (
"strings"
"time"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
@ -62,19 +65,48 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int
outputEvent[k] = v
}
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input)
gologger.Print().Msgf("%s", respBody)
}
event := createEvent(r, input, respBody, outputEvent)
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
}
callback(event)
return nil
}
// TODO extract duplicated code
func createEvent(request *Request, input string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
debugResponse := func(data string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
gologger.Print().Msgf("%s", data)
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
isMatch, matched := request.Match(data, matcher)
var result = response
if len(matched) != 0 {
if !request.options.Options.NoColor {
colorizer := aurora.NewAurora(true)
for _, currentMatch := range matched {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
debugResponse(result)
}
return isMatch, matched
}
result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = request.MakeResultEvent(event)
}
} else {
debugResponse(response)
}
return event
}

23
v2/pkg/protocols/http/operators.go
View File

@ -1,6 +1,7 @@
package http
import (
"fmt"
"net/http"
"strings"
"time"
@ -13,35 +14,35 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := getMatchPart(matcher.Part, data)
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
if !ok {
return false
return false, []string{}
}
status, ok := statusCode.(int)
if !ok {
return false
return false, []string{}
}
return matcher.Result(matcher.MatchStatusCode(status))
return matcher.Result(matcher.MatchStatusCode(status)), []string{fmt.Sprintf("HTTP/1.0 %d", status), fmt.Sprintf("HTTP/1.1 %d", status)}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(item)))
return matcher.Result(matcher.MatchSize(len(item))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(item, r.dynamicValues))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, r.dynamicValues))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

15
v2/pkg/protocols/http/operators_test.go
View File

@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}

63
v2/pkg/protocols/http/request.go
View File

@ -12,11 +12,13 @@ import (
"sync"
"time"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/multierr"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
@ -424,12 +426,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
}
}
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", string(redirectedResponse))
}
// if nuclei-project is enabled store the response if not previously done
if r.options.ProjectFile != nil && !fromcache {
if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil {
@ -467,20 +463,55 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
var ok bool
event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract)
if ok && event.OperatorsResult != nil {
event.OperatorsResult.PayloadValues = request.meta
event.Results = r.MakeResultEvent(event)
}
event.InternalEvent = outputEvent
}
event := createEvent(r, formedURL, outputEvent, string(redirectedResponse), finalEvent, request)
callback(event)
return nil
}
// TODO extract duplicated code
func createEvent(request *Request, formedURL string, outputEvent output.InternalEvent, response string, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent {
debugResponse := func(data string) {
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", data)
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if request.CompiledOperators != nil {
matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
isMatch, matched := request.Match(data, matcher)
//var result = data["response"].(string)
var result = response
if len(matched) != 0 {
if !request.options.Options.NoColor {
colorizer := aurora.NewAurora(true)
for _, currentMatch := range matched {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
debugResponse(result)
}
return isMatch, matched
}
result, ok := request.CompiledOperators.Execute(finalEvent, matcher, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = generatedRequest.meta
event.Results = request.MakeResultEvent(event)
}
} else {
debugResponse(response)
}
return event
}
// setCustomHeaders sets the custom headers for generated request
func (r *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range r.customHeaders {

16
v2/pkg/protocols/network/operators.go
View File

@ -11,7 +11,7 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "":
@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, r.dynamicValues))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

15
v2/pkg/protocols/network/operators_test.go
View File

@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}

58
v2/pkg/protocols/network/request.go
View File

@ -9,8 +9,11 @@ import (
"strings"
"time"
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
@ -187,11 +190,6 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
}
responseBuilder.Write(final[:n])
if r.options.Options.Debug || r.options.Options.DebugResponse {
responseOutput := responseBuilder.String()
gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput)))
}
outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress)
outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
for k, v := range previous {
@ -206,14 +204,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if interactURL == "" {
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = payloads
event.Results = r.MakeResultEvent(event)
}
}
event := createEvent(r, actualAddress, responseBuilder.String(), outputEvent, event, payloads)
callback(event)
} else if r.options.Interactsh != nil {
r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
@ -227,6 +218,47 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
return nil
}
// TODO extract duplicated code
func createEvent(request *Request, actualAddress string, response string, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent {
debugResponse := func(data string) {
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(response)))
}
}
if request.CompiledOperators != nil {
matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
isMatch, matched := request.Match(data, matcher)
//var result = data["response"].(string)
var result = response
if len(matched) != 0 {
if !request.options.Options.NoColor {
colorizer := aurora.NewAurora(true)
for _, currentMatch := range matched {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
debugResponse(result)
}
return isMatch, matched
}
result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = payloads
event.Results = request.MakeResultEvent(event)
}
} else {
debugResponse(response)
}
return event
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {

21
v2/pkg/protocols/offlinehttp/operators.go
View File

@ -1,6 +1,7 @@
package offlinehttp
import (
"fmt"
"net/http"
"strings"
"time"
@ -13,31 +14,31 @@ import (
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := getMatchPart(matcher.Part, data)
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
if !ok {
return false
return false, []string{}
}
return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
return matcher.Result(matcher.MatchStatusCode(statusCode.(int))), []string{fmt.Sprintf("HTTP/1.0 %d", statusCode), fmt.Sprintf("HTTP/1.1 %d", statusCode)}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(item)))
return matcher.Result(matcher.MatchSize(len(item))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(item, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.

15
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}

3
v2/pkg/protocols/offlinehttp/request.go
View File

@ -8,11 +8,12 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/remeh/sizedwaitgroup"
)
var _ protocols.Request = &Request{}

6
v2/pkg/protocols/protocols.go
View File

@ -74,8 +74,10 @@ type Request interface {
// condition matching. So, two requests can be sent and their match can
// be evaluated from the third request by using the IDs for both requests.
GetID() string
// Match performs matching operation for a matcher on model and returns true or false.
Match(data map[string]interface{}, matcher *matchers.Matcher) bool
// Match performs matching operation for a matcher on model and returns:
// true and a list of matched snippets if the matcher type is supports it
// otherwise false and an empty string slice
Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string)
// Extract performs extracting operation for an extractor on model and returns true or false.
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.