External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 03:25:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Do not show AND matcher information in the command line output if debug is not enabled #1081

Browse Source
This commit is contained in:
forgedhallpass 2021-10-12 20:06:55 +03:00
parent 8392143944
commit 435ec5cd5d
21 changed files with 165 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
v2/pkg/operators/operators.go
View File

@ -108,7 +108,7 @@ type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (boo
type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// Execute executes the operators on data and returns a result structure
func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) {
func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) {
matcherCondition := operators.GetMatchersCondition()
var matches bool
@ -140,9 +140,14 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
for matcherIndex, matcher := range operators.Matchers {
if isMatch, matched := match(data, matcher); isMatch {
matcherName := getMatcherName(matcher, matcherIndex)
result.Matches[matcherName] = matched
if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled
matcherName := getMatcherName(matcher, matcherIndex)
result.Matches[matcherName] = matched
} else { // if it's a "named" matcher with OR condition, then display it
if matcherCondition == matchers.ORCondition && matcher.Name != "" {
result.Matches[matcher.Name] = matched
}
}
matches = true
} else if matcherCondition == matchers.ANDCondition {
if len(result.DynamicValues) > 0 {

4
v2/pkg/protocols/common/clusterer/executer.go
View File

@ -67,7 +67,7 @@ func (e *Executer) Execute(input string) (bool, error) {
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
@ -98,7 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID

9
v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go
View File

@ -6,16 +6,17 @@ import (
)
// CreateEvent wraps the outputEvent with the result of the operators defined on the request
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent) *output.InternalWrappedEvent {
return CreateEventWithAdditionalOptions(request, outputEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {})
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent {
return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {})
}
// CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values.
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool,
addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
for _, compiledOperator := range request.GetCompiledOperators() {
if compiledOperator != nil {
result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract)
result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug)
if ok && result != nil {
event.OperatorsResult = result
addAdditionalOptions(event)

2
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -145,7 +145,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc)
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
if !matched || result == nil {
return false // if we don't match, return
}

67
v2/pkg/protocols/dns/operators.go
View File

@ -73,45 +73,20 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 11)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["request"] = req.String()
data["rcode"] = resp.Rcode
buffer := &bytes.Buffer{}
for _, question := range resp.Question {
buffer.WriteString(question.String())
return output.InternalEvent{
"host": host,
"matched": matched,
"request": req.String(),
"rcode": resp.Rcode,
"question": questionToString(resp.Question),
"extra": rrToString(resp.Extra),
"answer": rrToString(resp.Answer),
"ns": rrToString(resp.Ns),
"raw": resp.String(),
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
data["question"] = buffer.String()
buffer.Reset()
for _, extra := range resp.Extra {
buffer.WriteString(extra.String())
}
data["extra"] = buffer.String()
buffer.Reset()
for _, answer := range resp.Answer {
buffer.WriteString(answer.String())
}
data["answer"] = buffer.String()
buffer.Reset()
for _, ns := range resp.Ns {
buffer.WriteString(ns.String())
}
data["ns"] = buffer.String()
buffer.Reset()
rawData := resp.String()
data["raw"] = rawData
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
}
// MakeResultEvent creates a result event from internal wrapped event
@ -134,3 +109,19 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
}
return data
}
func rrToString(resourceRecords []dns.RR) string { // TODO rewrite with generics when available
buffer := &bytes.Buffer{}
for _, resourceRecord := range resourceRecords {
buffer.WriteString(resourceRecord.String())
}
return buffer.String()
}
func questionToString(resourceRecords []dns.Question) string {
buffer := &bytes.Buffer{}
for _, resourceRecord := range resourceRecords {
buffer.WriteString(resourceRecord.String())
}
return buffer.String()
}

8
v2/pkg/protocols/dns/operators_test.go
View File

@ -236,13 +236,15 @@ func TestDNSMakeResult(t *testing.T) {
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
resultEvent := finalEvent.Results[0]
require.Equal(t, "test", resultEvent.MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
require.Equal(t, "one.one.one.one", resultEvent.Matched, "could not get matched value")
}

4
v2/pkg/protocols/dns/request.go
View File

@ -8,9 +8,9 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
)
var _ protocols.Request = &Request{}
@ -62,7 +62,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)

22
v2/pkg/protocols/file/operators.go
View File

@ -66,18 +66,16 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (request *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 5)
// Some data regarding the request metadata
data["path"] = host
data["matched"] = matched
data["raw"] = raw
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
// responseToDSLMap converts a file response to a map for use in DSL matching
func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent {
return output.InternalEvent{
"path": inputFilePath,
"matched": matchedFileName,
"raw": raw,
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event

83
v2/pkg/protocols/file/operators_test.go
View File

@ -159,7 +159,62 @@ func TestFileOperatorExtract(t *testing.T) {
})
}
func TestFileMakeResult(t *testing.T) {
func TestFileMakeResultWithOrMatcher(t *testing.T) {
expectedValue := []string{"1.1.1.1"}
namedMatcherName := "test"
finalEvent := testFileMakeResultOperators(t, "or")
require.Equal(t, namedMatcherName, finalEvent.Results[0].MatcherName)
require.Equal(t, expectedValue, finalEvent.OperatorsResult.Matches[namedMatcherName], "could not get matched value")
}
func TestFileMakeResultWithAndMatcher(t *testing.T) {
finalEvent := testFileMakeResultOperators(t, "and")
require.Equal(t, "", finalEvent.Results[0].MatcherName)
require.Empty(t, finalEvent.OperatorsResult.Matches)
}
func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.InternalWrappedEvent {
expectedValue := []string{"1.1.1.1"}
namedMatcherName := "test"
matcher := []*matchers.Matcher{
{
Part: "raw",
Type: "word",
Words: expectedValue,
},
{
Name: namedMatcherName,
Part: "raw",
Type: "word",
Words: expectedValue,
},
}
expectedValues := map[string][]string{
"word-1": expectedValue,
namedMatcherName: expectedValue,
}
finalEvent := testFileMakeResult(t, matcher, matcherCondition, true)
for matcherName, matchedValues := range expectedValues {
var matchesOne = false
for i := 0; i <= len(expectedValue); i++ {
resultEvent := finalEvent.Results[i]
if matcherName == resultEvent.MatcherName {
matchesOne = true
}
}
require.True(t, matchesOne)
require.Equal(t, matchedValues, finalEvent.OperatorsResult.Matches[matcherName], "could not get matched value")
}
finalEvent = testFileMakeResult(t, matcher, matcherCondition, false)
require.Equal(t, 1, len(finalEvent.Results))
return finalEvent
}
func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondition string, isDebug bool) *output.InternalWrappedEvent {
options := testutils.DefaultOptions
testutils.Init(options)
@ -171,12 +226,8 @@ func TestFileMakeResult(t *testing.T) {
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}},
MatchersCondition: matcherCondition,
Matchers: matchers,
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
@ -191,20 +242,24 @@ func TestFileMakeResult(t *testing.T) {
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
matchedFileName := "test.txt"
fileContent := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName)
require.Len(t, event, 6, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
require.Equal(t, fileContent, event["raw"], "could not get correct resp")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
resultEvent := finalEvent.Results[0]
require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value")
return finalEvent
}

2
v2/pkg/protocols/file/request.go
View File

@ -57,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)

22
v2/pkg/protocols/headless/operators.go
View File

@ -64,19 +64,17 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
// responseToDSLMap converts a headless response to a map for use in DSL matching
func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 5)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["req"] = req
data["data"] = resp
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
return output.InternalEvent{
"host": host,
"matched": matched,
"req": req,
"data": resp,
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event

2
v2/pkg/protocols/headless/request.go
View File

@ -65,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)

4
v2/pkg/protocols/http/operators.go
View File

@ -97,8 +97,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) {
}
// responseToDSLMap converts an HTTP response to a map for use in DSL matching
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies()))
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}

2
v2/pkg/protocols/http/operators_test.go
View File

@ -262,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) {
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

2
v2/pkg/protocols/http/request.go
View File

@ -477,7 +477,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
}
}
event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, func(internalWrappedEvent *output.InternalWrappedEvent) {
event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})

24
v2/pkg/protocols/network/operators.go
View File

@ -64,20 +64,18 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
// responseToDSLMap converts a network response to a map for use in DSL matching
func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 6)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["request"] = req
data["data"] = resp // Data is the last bytes read
data["raw"] = raw // Raw is the full transaction data for network
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
return output.InternalEvent{
"host": host,
"matched": matched,
"request": req,
"data": resp, // Data is the last bytes read
"raw": raw, // Raw is the full transaction data for network
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event

2
v2/pkg/protocols/network/operators_test.go
View File

@ -192,7 +192,7 @@ func TestNetworkMakeResult(t *testing.T) {
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
event["ip"] = "192.168.1.1"
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

2
v2/pkg/protocols/network/request.go
View File

@ -209,7 +209,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input
var event *output.InternalWrappedEvent
if interactURL == "" {
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, func(wrappedEvent *output.InternalWrappedEvent) {
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payloads
})
callback(event)

4
v2/pkg/protocols/offlinehttp/operators.go
View File

@ -93,8 +93,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) {
}
// responseToDSLMap converts an HTTP response to a map for use in DSL matching
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies()))
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}

2
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -204,7 +204,7 @@ func TestHTTPMakeResult(t *testing.T) {
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
for _, operator := range request.compiledOperators {
result, ok := operator.Execute(event, request.Match, request.Extract)
result, ok := operator.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

2
v2/pkg/protocols/offlinehttp/request.go
View File

@ -85,7 +85,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v
}
event := eventcreator.CreateEvent(request, outputEvent)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
callback(event)
}(data)
})