diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 7f24385fd..116479732 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -85,6 +85,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), + flagSet.BoolVarP(&options.MatcherStatus, "matcher-status", "ms", false, "show optional match failure status"), flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a1d6e716..055eca1ab 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -117,7 +117,7 @@ func New(options *types.Options) (*Runner, error) { runner.hmapInputProvider = hmapInput // Create the output file if asked - outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile, options.ErrorLogFile) + outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.Output, options.TraceLogFile, options.ErrorLogFile) if err != nil { return nil, errors.Wrap(err, "could not create output file") } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index ddab852ae..2331095d0 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -27,6 +27,15 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.aurora.BrightGreen(output.ExtractorName).Bold().String()) } + if w.matcherStatus { + builder.WriteString("] [") + if !output.MatcherStatus { + builder.WriteString(w.aurora.Red("failed").String()) + } else { + builder.WriteString(w.aurora.Green("matched").String()) + } + } + builder.WriteString("] [") builder.WriteString(w.aurora.BrightBlue(output.Type).String()) builder.WriteString("] ") @@ -35,7 +44,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity)) builder.WriteString("] ") } - builder.WriteString(output.Matched) + if output.Matched != "" { + builder.WriteString(output.Matched) + } else { + builder.WriteString(output.Host) + } // If any extractors, write the results if len(output.ExtractedResults) > 0 { diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 991c99232..e78699f7a 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) @@ -27,6 +28,8 @@ type Writer interface { Colorizer() aurora.Aurora // Write writes the event to file and/or screen. Write(*ResultEvent) error + // WriteFailure writes the optional failure event for template to file and/or screen. + WriteFailure(event InternalEvent) error // Request logs a request in the trace log Request(templateID, url, requestType string, err error) } @@ -37,6 +40,7 @@ type StandardWriter struct { jsonReqResp bool noTimestamp bool noMetadata bool + matcherStatus bool aurora aurora.Aurora outputFile io.WriteCloser traceFile io.WriteCloser @@ -54,10 +58,16 @@ type InternalWrappedEvent struct { InternalEvent InternalEvent Results []*ResultEvent OperatorsResult *operators.Result + UsesInteractsh bool } // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { + // Template is the relative filename for the template + Template string `json:"template,omitempty"` + // TemplateURL is the URL of the template for the result inside the nuclei + // templates repository if it belongs to the repository. + TemplateURL string `json:"template-url,omitempty"` // TemplateID is the ID of the template for the result. TemplateID string `json:"template-id"` // TemplatePath is the path of template @@ -92,12 +102,14 @@ type ResultEvent struct { Interaction *server.Interaction `json:"interaction,omitempty"` // CURLCommand is an optional curl command to reproduce the request // Only applicable if the report is for HTTP. - CURLCommand string `json:"curl-command,omitempty"` + CURLCommand string `json:"curl-command,omitempty"` + // MatcherStatus is the status of the match + MatcherStatus bool `json:"matcher-status"` FileToIndexPosition map[string]int `json:"-"` } // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) var outputFile io.WriteCloser @@ -128,6 +140,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, json: json, jsonReqResp: jsonReqResp, noMetadata: noMetadata, + matcherStatus: MatcherStatus, noTimestamp: noTimestamp, aurora: auroraColorizer, outputFile: outputFile, @@ -140,6 +153,10 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, // Write writes the event to file and/or screen. func (w *StandardWriter) Write(event *ResultEvent) error { + // Enrich the result event with extra metadata on the template-path and url. + if event.TemplatePath != "" { + event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath)) + } event.Timestamp = time.Now() var data []byte @@ -224,3 +241,23 @@ func (w *StandardWriter) Close() { w.errorFile.Close() } } + +// WriteFailure writes the failure event for template to file and/or screen. +func (w *StandardWriter) WriteFailure(event InternalEvent) error { + if !w.matcherStatus { + return nil + } + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"])) + data := &ResultEvent{ + Template: templatePath, + TemplateURL: templateURL, + TemplateID: types.ToString(event["template-id"]), + TemplatePath: types.ToString(event["template-path"]), + Info: event["template-info"].(model.Info), + Type: types.ToString(event["type"]), + Host: types.ToString(event["host"]), + MatcherStatus: false, + Timestamp: time.Now(), + } + return w.Write(data) +} diff --git a/v2/pkg/output/output_test.go b/v2/pkg/output/output_test.go index 9f1d188bb..1648dcc36 100644 --- a/v2/pkg/output/output_test.go +++ b/v2/pkg/output/output_test.go @@ -11,7 +11,7 @@ import ( func TestStandardWriterRequest(t *testing.T) { t.Run("WithoutTraceAndError", func(t *testing.T) { - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") require.NoError(t, err) require.NotPanics(t, func() { w.Request("path", "input", "http", nil) @@ -23,7 +23,7 @@ func TestStandardWriterRequest(t *testing.T) { traceWriter := &testWriteCloser{} errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.traceFile = traceWriter w.errorFile = errorWriter require.NoError(t, err) @@ -36,7 +36,7 @@ func TestStandardWriterRequest(t *testing.T) { t.Run("ErrorWithWrappedError", func(t *testing.T) { errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.errorFile = errorWriter require.NoError(t, err) w.Request( diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 3aaf774cb..fe650e945 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" ) // Executer executes a group of requests for a protocol @@ -59,18 +60,17 @@ func (e *Executer) Execute(input string) (bool, error) { builder.Reset() } } - if event.OperatorsResult == nil { - return - } - for _, result := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if event.OperatorsResult == nil && !event.UsesInteractsh { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + } else { + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results = true } - results = true - _ = e.options.Output.Write(result) - e.options.Progress.IncrementMatched() } }) if err != nil { diff --git a/v2/pkg/protocols/common/helpers/writer/writer.go b/v2/pkg/protocols/common/helpers/writer/writer.go new file mode 100644 index 000000000..91b98f33b --- /dev/null +++ b/v2/pkg/protocols/common/helpers/writer/writer.go @@ -0,0 +1,35 @@ +package writer + +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" +) + +// WriteResult is a helper for writing results to the output +func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool { + // Handle the case where no result found for the template. + // In this case, we just show misc information about the failed + // match for the template. + if data.OperatorsResult == nil { + return false + } + var matched bool + for _, result := range data.Results { + if err := output.Write(result); err != nil { + gologger.Warning().Msgf("Could not write output event: %s\n", err) + } + if !matched { + matched = true + } + progress.IncrementMatched() + + if issuesClient != nil { + if err := issuesClient.CreateIssue(result); err != nil { + gologger.Warning().Msgf("Could not create issue on tracker: %s", err) + } + } + } + return matched +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 7c3e02f82..b810c346d 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" ) @@ -177,19 +178,8 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d } data.Event.Results = data.MakeResultFunc(data.Event) - for _, result := range data.Event.Results { - result.Interaction = interaction - _ = c.options.Output.Write(result) - if !c.matched { - c.matched = true - } - c.options.Progress.IncrementMatched() - - if c.options.IssuesClient != nil { - if err := c.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } + if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { + c.matched = true } return true } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 12de136a8..0c1ff929c 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -90,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, + "type": request.Type().String(), "trace": traceToString(tracedata, false), } } @@ -104,10 +105,11 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "dns", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, Timestamp: time.Now(), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index aef43ccfa..5651ea422 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -45,7 +45,7 @@ func TestResponseToDSLMap(t *testing.T) { resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode") } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index cde2e63e0..41ef7fd3a 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName str "path": inputFilePath, "matched": matchedFileName, "raw": raw, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -120,10 +121,11 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ + MatcherStatus: true, TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "file", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index ac41a9400..ffdddc083 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { resp := "test-data\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") } @@ -60,7 +60,7 @@ func TestFileOperatorMatch(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("valid", func(t *testing.T) { @@ -109,7 +109,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { resp := "TEST-DATA\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") matcher := &matchers.Matcher{ @@ -148,7 +148,7 @@ func TestFileOperatorExtract(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("extract", func(t *testing.T) { @@ -266,7 +266,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi fileContent := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 3d62e61b5..b9f9b4ccc 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -72,6 +72,7 @@ func (request *Request) responseToDSLMap(resp, req, host, matched string) output "matched": matched, "req": req, "data": resp, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -92,11 +93,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "headless", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 868fd0118..1f319ccff 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data[k] = strings.Join(v, " ") } data["host"] = host + data["type"] = request.Type().String() data["matched"] = matched data["request"] = rawReq data["response"] = rawResp @@ -141,12 +142,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 7aefd7d19..ae6c644cf 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -71,7 +71,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -159,7 +159,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header") @@ -286,7 +286,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 94782c2a9..8f2040782 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -473,6 +473,9 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) + if hasInteractMarkers { + event.UsesInteractsh = true + } responseContentType := resp.Header.Get("Content-Type") dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType) diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c829b8ade..010b11d61 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) o "request": req, "data": resp, // Data is the last bytes read "raw": raw, // Raw is the full transaction data for network + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -93,12 +94,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "network", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Metadata: wrapped.OperatorsResult.PayloadValues, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index daa92725e..805c0ed22 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { req := "test-data\r\n" resp := "resp-data\r\n" event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one") - require.Len(t, event, 8, "could not get correct number of items in dsl map") + require.Len(t, event, 9, "could not get correct number of items in dsl map") require.Equal(t, resp, event["data"], "could not get correct resp") } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ae63dc0d0..35da81d15 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -281,6 +281,9 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input ExtractFunc: request.Extract, }) } + if len(interactshURLs) > 0 { + event.UsesInteractsh = true + } dumpResponse(event, request.options, response, actualAddress) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index b5141037b..203693090 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body + data["type"] = request.Type().String() data["all_headers"] = headers data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID @@ -135,11 +136,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index b69ccbdc2..0a3a663f0 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -37,7 +37,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -63,7 +63,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -132,7 +132,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header") @@ -198,7 +198,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index ba26c8e30..388886be2 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -129,6 +129,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] + data["type"] = request.Type().String() data["response"] = jsonDataString data["host"] = input data["matched"] = addressToDial @@ -195,12 +196,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), } return data diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 47a87a78d..ab864f65b 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -248,6 +248,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam for k, v := range events { data[k] = v } + + data["type"] = request.Type().String() data["success"] = "true" data["request"] = requestOutput data["response"] = responseBuilder.String() @@ -364,12 +366,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 7e0ed1cf2..1ec1e95cb 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/rs/xid" ) @@ -147,22 +148,22 @@ func (e *Executer) Execute(input string) (bool, error) { err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) + event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath + event.InternalEvent["template-info"] = operator.templateInfo + + if result == nil && !matched { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + continue + } if matched && result != nil { event.OperatorsResult = result - event.InternalEvent["template-id"] = operator.templateID - event.InternalEvent["template-path"] = operator.templatePath - event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) results = true - for _, r := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(r); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } - _ = e.options.Output.Write(r) - e.options.Progress.IncrementMatched() - } + + _ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) } } }) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 8e0c5923f..982fbabc3 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -131,6 +131,11 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro } } +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { + return nil +} + type MockProgressClient struct{} // Stop stops the progress recorder. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c2619b7be..1a0935338 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -177,6 +177,8 @@ type Options struct { NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool + // MatcherStatus displays optional status for the failed matches as well + MatcherStatus bool // ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts ClientCertFile string // ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts diff --git a/v2/pkg/utils/template_path.go b/v2/pkg/utils/template_path.go new file mode 100644 index 000000000..2039c2d97 --- /dev/null +++ b/v2/pkg/utils/template_path.go @@ -0,0 +1,32 @@ +package utils + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" +) + +const ( + // TemplatesRepoURL is the URL for files in nuclei-templates repository + TemplatesRepoURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master/" +) + +var configData *config.Config + +func init() { + configData, _ = config.ReadConfiguration() +} + +// TemplatePathURL returns the Path and URL for the provided template +func TemplatePathURL(fullPath string) (string, string) { + var templateDirectory string + if configData != nil && configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { + templateDirectory = configData.TemplatesDirectory + } else { + return "", "" + } + + finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/") + templateURL := TemplatesRepoURL + finalPath + return finalPath, templateURL +}