diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 401110cfb..00c01d1ed 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -40,7 +40,8 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool { +func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { if dynamicValues == nil { @@ -57,7 +58,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -65,19 +66,21 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{word} } + matchedWords = append(matchedWords, word) + // If we are at the end of the words, return with true if len(m.Words)-1 == i { - return true + return true, matchedWords } } - return false + return false, []string{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) bool { +func (m *Matcher) MatchRegex(corpus string) (bool, []string) { // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -85,7 +88,7 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -93,19 +96,19 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, regex.FindAllString(corpus, -1) } // If we are at the end of the regex, return with true if len(m.regexCompiled)-1 == i { - return true + return true, []string{corpus} } } - return false + return false, []string{} } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) bool { +func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match @@ -114,7 +117,7 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -122,15 +125,15 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{string(hexa)} } // If we are at the end of the words, return with true if len(m.Binary)-1 == i { - return true + return true, []string{string(hexa)} } } - return false + return false, []string{} } // MatchDSL matches on a generic map result diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 6a2d0b858..d5161ca61 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -9,24 +9,29 @@ import ( func TestANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid AND condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid AND condition") + require.Equal(t, m.Words, matched) - matched = m.MatchWords("b", nil) - require.False(t, matched, "Could match invalid AND condition") + isMatched, matched = m.MatchWords("b", nil) + require.False(t, isMatched, "Could match invalid AND condition") + require.Equal(t, []string{}, matched) } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid OR condition") + require.Equal(t, []string{"a"}, matched) - matched = m.MatchWords("b", nil) - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched = m.MatchWords("b", nil) + require.True(t, isMatched, "Could not match valid OR condition") + require.Equal(t, []string{"b"}, matched) - matched = m.MatchWords("c", nil) - require.False(t, matched, "Could match invalid OR condition") + isMatched, matched = m.MatchWords("c", nil) + require.False(t, isMatched, "Could match invalid OR condition") + require.Equal(t, []string{}, matched) } func TestHexEncoding(t *testing.T) { @@ -34,6 +39,7 @@ func TestHexEncoding(t *testing.T) { err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := m.MatchWords("PING", nil) - require.True(t, matched, "Could not match valid Hex condition") + isMatched, matched := m.MatchWords("PING", nil) + require.True(t, isMatched, "Could not match valid Hex condition") + require.Equal(t, m.Words, matched) } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index a0d176477..88ee413e1 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool { return data } +// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string +func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if m.Negative { + return !data, []string{} + } + return data, matchedSnippet +} + // GetType returns the type of the matcher func (m *Matcher) GetType() MatcherType { return m.matcherType diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 493eddf06..23ed85978 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -100,7 +100,7 @@ func (r *Result) Merge(result *Result) { } // MatchFunc performs matching operation for a matcher on model and returns true or false. -type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} @@ -138,21 +138,19 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac for _, matcher := range r.Matchers { // Check if the matcher matched - if !match(data, matcher) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - if len(result.DynamicValues) > 0 { - return result, true - } - return nil, false - } - } else { + if isMatch, _ := match(data, matcher); isMatch { // If the matcher has matched, and it's an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && matcher.Name != "" { result.Matches[matcher.Name] = struct{}{} } + matches = true + } else if matcherCondition == matchers.ANDCondition { + if len(result.DynamicValues) > 0 { + return result, true + } + return nil, false } } @@ -162,7 +160,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return result, true } - // Don't print if we have matchers and they have not matched, regardless of extractor + // Don't print if we have matchers, and they have not matched, regardless of extractor if len(r.Matchers) > 0 && !matches { return nil, false } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index cda725d33..3b86e234b 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -14,7 +14,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -23,24 +23,24 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } switch matcher.GetType() { - case matchers.StatusMatcher: - return matcher.Result(matcher.MatchStatusCode(item.(int))) + case matchers.StatusMatcher: // TODO is this correct? + return matcher.Result(matcher.MatchStatusCode(item.(int))), []string{} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) + return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item), nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item))) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 286dd582f..d249f1953 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("rcode", func(t *testing.T) { @@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile rcode matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid rcode response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid rcode response") + require.Equal(t, []string{}, matched) }) t.Run("negative", func(t *testing.T) { @@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ae3041f33..f6487603a 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -2,9 +2,13 @@ package dns import ( "net/url" + "strings" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -48,27 +52,57 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused r.options.Output.Request(r.options.TemplateID, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", resp.String()) - } outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, domain, resp.String(), outputEvent) + callback(event) return nil } +// TODO extract duplicated code +func createEvent(request *Request, domain string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result string = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 743dd7d13..11a804cab 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -13,7 +13,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "data", "": @@ -22,23 +22,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 7c5be06d9..7b985f86b 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index d272e256b..fe2342dc6 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -3,13 +3,17 @@ package file import ( "io/ioutil" "os" + "strings" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} @@ -21,50 +25,40 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused err := r.getInputPaths(input, func(data string) { wg.Add() - go func(data string) { + go func(filePath string) { defer wg.Done() - file, err := os.Open(data) + file, err := os.Open(filePath) if err != nil { - gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err) return } defer file.Close() stat, err := file.Stat() if err != nil { - gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err) return } if stat.Size() >= int64(r.MaxSize) { - gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath) return } buffer, err := ioutil.ReadAll(file) if err != nil { - gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } dataStr := tostring.UnsafeToString(buffer) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) - gologger.Print().Msgf("%s", dataStr) - } - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - outputEvent := r.responseToDSLMap(dataStr, input, data) + + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, filePath) + outputEvent := r.responseToDSLMap(dataStr, input, filePath) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, filePath, dataStr, outputEvent) callback(event) }(data) }) @@ -77,3 +71,43 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused r.options.Progress.IncrementRequests() return nil } + +// TODO extract duplicated code +func createEvent(request *Request, filePath string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index c384ee26a..1053acccb 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "resp", "": @@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 54345d468..095b7afdf 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -5,8 +5,11 @@ import ( "strings" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -62,19 +65,48 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int outputEvent[k] = v } - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", respBody) - } + event := createEvent(r, input, respBody, outputEvent) - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } callback(event) return nil } + +// TODO extract duplicated code +func createEvent(request *Request, input string, response string, outputEvent output.InternalEvent) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a836fba7b..b35bae8e4 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -1,6 +1,7 @@ package http import ( + "fmt" "net/http" "strings" "time" @@ -13,35 +14,35 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: statusCode, ok := data["status_code"] if !ok { - return false + return false, []string{} } status, ok := statusCode.(int) if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(status)) + return matcher.Result(matcher.MatchStatusCode(status)), []string{fmt.Sprintf("HTTP/1.0 %d", status), fmt.Sprintf("HTTP/1.1 %d", status)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, r.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 6c24f236a..509312ea1 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d8a2c98dc..a6c493d69 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -12,11 +12,13 @@ import ( "sync" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -424,12 +426,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", string(redirectedResponse)) - } - // if nuclei-project is enabled store the response if not previously done if r.options.ProjectFile != nil && !fromcache { if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { @@ -467,20 +463,55 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult.PayloadValues = request.meta - event.Results = r.MakeResultEvent(event) - } - event.InternalEvent = outputEvent - } + event := createEvent(r, formedURL, outputEvent, string(redirectedResponse), finalEvent, request) + callback(event) return nil } +// TODO extract duplicated code +func createEvent(request *Request, formedURL string, outputEvent output.InternalEvent, response string, finalEvent output.InternalEvent, generatedRequest *generatedRequest) *output.InternalWrappedEvent { + debugResponse := func(data string) { + // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", data) + } + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + //var result = data["response"].(string) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(finalEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = generatedRequest.meta + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // setCustomHeaders sets the custom headers for generated request func (r *Request) setCustomHeaders(req *generatedRequest) { for k, v := range r.customHeaders { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 75e71dda7..9bb996bca 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -11,7 +11,7 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -20,23 +20,23 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, r.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 4c8ea3028..474ac0dec 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 61abdf3fc..e64a292bd 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -9,8 +9,11 @@ import ( "strings" "time" + "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -187,11 +190,6 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } responseBuilder.Write(final[:n]) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput))) - } outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress) outputEvent["ip"] = r.dialer.GetDialedIP(hostname) for k, v := range previous { @@ -206,14 +204,7 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if interactURL == "" { - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = payloads - event.Results = r.MakeResultEvent(event) - } - } + event := createEvent(r, actualAddress, responseBuilder.String(), outputEvent, event, payloads) callback(event) } else if r.options.Interactsh != nil { r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ @@ -227,6 +218,47 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin return nil } +// TODO extract duplicated code +func createEvent(request *Request, actualAddress string, response string, outputEvent output.InternalEvent, event *output.InternalWrappedEvent, payloads map[string]interface{}) *output.InternalWrappedEvent { + debugResponse := func(data string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, hex.EncodeToString([]byte(response))) + } + } + + if request.CompiledOperators != nil { + + matcher := func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + isMatch, matched := request.Match(data, matcher) + //var result = data["response"].(string) + var result = response + + if len(matched) != 0 { + if !request.options.Options.NoColor { + colorizer := aurora.NewAurora(true) + for _, currentMatch := range matched { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + debugResponse(result) + } + + return isMatch, matched + } + + result, ok := request.CompiledOperators.Execute(outputEvent, matcher, request.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.OperatorsResult.PayloadValues = payloads + event.Results = request.MakeResultEvent(event) + } + } else { + debugResponse(response) + } + return event +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index ace976856..c5a7352da 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -1,6 +1,7 @@ package offlinehttp import ( + "fmt" "net/http" "strings" "time" @@ -13,31 +14,31 @@ import ( ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: statusCode, ok := data["status_code"] if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + return matcher.Result(matcher.MatchStatusCode(statusCode.(int))), []string{fmt.Sprintf("HTTP/1.0 %d", statusCode), fmt.Sprintf("HTTP/1.1 %d", statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item, nil)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 56facee78..eba24203e 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 27b10e9db..9404180c1 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -8,11 +8,12 @@ import ( "strings" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333e..2e03434fd 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -74,8 +74,10 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Match performs matching operation for a matcher on model and returns: + // true and a list of matched snippets if the matcher type is supports it + // otherwise false and an empty string slice + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them.