diff --git a/v2/go.mod b/v2/go.mod index b138fdbbe..51edf74ac 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -43,7 +43,7 @@ require ( github.com/valyala/fasttemplate v1.2.2 github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 github.com/xanzy/go-gitlab v0.79.0 - go.uber.org/multierr v1.9.0 + go.uber.org/multierr v1.9.0 // indirect golang.org/x/net v0.5.0 golang.org/x/oauth2 v0.4.0 golang.org/x/text v0.6.0 diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8a3d01ffe..6fb109491 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -70,7 +70,7 @@ type Runner struct { catalog catalog.Catalog progress progress.Progress colorizer aurora.Aurora - issuesClient *reporting.Client + issuesClient reporting.Client hmapInputProvider *hybrid.Input browser *engine.Browser ratelimiter *ratelimit.Limiter diff --git a/v2/pkg/protocols/common/helpers/writer/writer.go b/v2/pkg/protocols/common/helpers/writer/writer.go index 91b98f33b..c81b7d3d2 100644 --- a/v2/pkg/protocols/common/helpers/writer/writer.go +++ b/v2/pkg/protocols/common/helpers/writer/writer.go @@ -8,7 +8,7 @@ import ( ) // WriteResult is a helper for writing results to the output -func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool { +func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool { // Handle the case where no result found for the template. // In this case, we just show misc information about the failed // match for the template. diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 40c212b8f..aa322d18b 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -84,7 +84,7 @@ type Options struct { // Output is the output writer for nuclei Output output.Writer // IssuesClient is a client for issue exporting - IssuesClient *reporting.Client + IssuesClient reporting.Client // Progress is the nuclei progress bar implementation. Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client @@ -132,7 +132,7 @@ func New(options *Options) (*Client, error) { } // NewDefaultOptions returns the default options for interactsh client -func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { +func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options { return &Options{ ServerURL: client.DefaultOptions.ServerURL, CacheSize: 5000, diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1bd348840..b5c9110b1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -13,9 +13,9 @@ import ( "sync" "time" - "github.com/pkg/errors" + "errors" + "github.com/remeh/sizedwaitgroup" - "go.uber.org/multierr" "moul.io/http2curl" "github.com/projectdiscovery/gologger" @@ -36,6 +36,7 @@ import ( templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" + errorutil "github.com/projectdiscovery/utils/errors" stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" ) @@ -107,7 +108,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { - requestErr = multierr.Append(requestErr, err) + requestErr = errors.Join(requestErr, err) } mutex.Unlock() }(generatedRequests[i]) @@ -154,7 +155,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { - requestErr = multierr.Append(requestErr, err) + requestErr = errors.Join(requestErr, err) } mutex.Unlock() }(generatedHttpRequest) @@ -217,7 +218,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { - requestErr = multierr.Append(requestErr, err) + requestErr = errors.Join(requestErr, err) } mutex.Unlock() }(generatedHttpRequest) @@ -231,7 +232,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { parsed, err := urlutil.Parse(input.MetaInput.Input) if err != nil { - return errors.Wrap(err, "could not parse url") + return errorutil.NewWithErr(err).Msgf("could not parse url") } fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) @@ -304,7 +305,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous return nil } if err != nil { - return errors.Wrap(err, "could not execute rule") + return errorutil.NewWithErr(err).Msgf("could not execute rule") } } } @@ -558,7 +559,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ connConfiguration.Connection.Cookiejar = input.CookieJar client, err := httpclientpool.Get(request.options.Options, connConfiguration) if err != nil { - return errors.Wrap(err, "could not get http client") + return errorutil.NewWithErr(err).Msgf("could not get http client") } httpclient = client } @@ -645,7 +646,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ dumpedResponseHeaders, err := httputil.DumpResponse(resp, false) if err != nil { - return errors.Wrap(err, "could not dump http response") + return errorutil.NewWithErr(err).Msgf("could not dump http response") } var dumpedResponse []redirectedResponse @@ -666,7 +667,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error()) } else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error - return errors.Wrap(err, "could not read http body") + return errorutil.NewWithErr(err).Msgf("could not read http body") } } gotData = data @@ -674,7 +675,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) if err != nil { - return errors.Wrap(err, "could not read http response with redirect chain") + return errorutil.NewWithErr(err).Msgf("could not read http response with redirect chain") } } else { dumpedResponse = []redirectedResponse{{resp: resp, fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} @@ -683,7 +684,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // if nuclei-project is enabled store the response if not previously done if request.options.ProjectFile != nil && !fromCache { if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil { - return errors.Wrap(err, "could not store in project file") + return errorutil.NewWithErr(err).Msgf("could not store in project file") } } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index f9e4e03aa..a5353d16b 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -50,7 +50,7 @@ type ExecuterOptions struct { // Options contains configuration options for the executer. Options *types.Options // IssuesClient is a client for nuclei issue tracker reporting - IssuesClient *reporting.Client + IssuesClient reporting.Client // Progress is a progress client for scan reporting Progress progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. diff --git a/v2/pkg/reporting/client.go b/v2/pkg/reporting/client.go new file mode 100644 index 000000000..582e30657 --- /dev/null +++ b/v2/pkg/reporting/client.go @@ -0,0 +1,15 @@ +package reporting + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" +) + +// Client is a client for nuclei issue tracking module +type Client interface { + RegisterTracker(tracker Tracker) + RegisterExporter(exporter Exporter) + Close() + Clear() + CreateIssue(event *output.ResultEvent) error + GetReportingOptions() *Options +} diff --git a/v2/pkg/reporting/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go index c56e2205b..3fabc9c92 100644 --- a/v2/pkg/reporting/dedupe/dedupe.go +++ b/v2/pkg/reporting/dedupe/dedupe.go @@ -51,6 +51,18 @@ func New(dbPath string) (*Storage, error) { return storage, nil } +func (s *Storage) Clear() { + var keys [][]byte + iter := s.storage.NewIterator(nil, nil) + for iter.Next() { + keys = append(keys, iter.Key()) + } + iter.Release() + for _, key := range keys { + s.storage.Delete(key, nil) + } +} + // Close closes the storage for further operations func (s *Storage) Close() { s.storage.Close() diff --git a/v2/pkg/reporting/options.go b/v2/pkg/reporting/options.go new file mode 100644 index 000000000..1397f47b2 --- /dev/null +++ b/v2/pkg/reporting/options.go @@ -0,0 +1,36 @@ +package reporting + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/splunk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" + "github.com/projectdiscovery/retryablehttp-go" +) + +// Options is a configuration file for nuclei reporting module +type Options struct { + // AllowList contains a list of allowed events for reporting module + AllowList *Filter `yaml:"allow-list"` + // DenyList contains a list of denied events for reporting module + DenyList *Filter `yaml:"deny-list"` + // GitHub contains configuration options for GitHub Issue Tracker + GitHub *github.Options `yaml:"github"` + // GitLab contains configuration options for GitLab Issue Tracker + GitLab *gitlab.Options `yaml:"gitlab"` + // Jira contains configuration options for Jira Issue Tracker + Jira *jira.Options `yaml:"jira"` + // MarkdownExporter contains configuration options for Markdown Exporter Module + MarkdownExporter *markdown.Options `yaml:"markdown"` + // SarifExporter contains configuration options for Sarif Exporter Module + SarifExporter *sarif.Options `yaml:"sarif"` + // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module + ElasticsearchExporter *es.Options `yaml:"elasticsearch"` + // SplunkExporter contains configuration options for splunkhec Exporter Module + SplunkExporter *splunk.Options `yaml:"splunkhec"` + + HttpClient *retryablehttp.Client `yaml:"-"` +} diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 3610b7437..3ca188321 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -3,12 +3,11 @@ package reporting import ( "os" "path/filepath" - "strings" - "github.com/pkg/errors" - "go.uber.org/multierr" "gopkg.in/yaml.v2" + "errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" @@ -21,34 +20,11 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" - "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + sliceutil "github.com/projectdiscovery/utils/slice" ) -// Options is a configuration file for nuclei reporting module -type Options struct { - // AllowList contains a list of allowed events for reporting module - AllowList *Filter `yaml:"allow-list"` - // DenyList contains a list of denied events for reporting module - DenyList *Filter `yaml:"deny-list"` - // GitHub contains configuration options for GitHub Issue Tracker - GitHub *github.Options `yaml:"github"` - // GitLab contains configuration options for GitLab Issue Tracker - GitLab *gitlab.Options `yaml:"gitlab"` - // Jira contains configuration options for Jira Issue Tracker - Jira *jira.Options `yaml:"jira"` - // MarkdownExporter contains configuration options for Markdown Exporter Module - MarkdownExporter *markdown.Options `yaml:"markdown"` - // SarifExporter contains configuration options for Sarif Exporter Module - SarifExporter *sarif.Options `yaml:"sarif"` - // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module - ElasticsearchExporter *es.Options `yaml:"elasticsearch"` - // SplunkExporter contains configuration options for splunkhec Exporter Module - SplunkExporter *splunk.Options `yaml:"splunkhec"` - - HttpClient *retryablehttp.Client `yaml:"-"` -} - // Filter filters the received event and decides whether to perform // reporting for it or not. type Filter struct { @@ -56,9 +32,9 @@ type Filter struct { Tags stringslice.StringSlice `yaml:"tags"` } -const ( - reportingClientCreationErrorMessage = "could not create reporting client" - exportClientCreationErrorMessage = "could not create exporting client" +var ( + ErrReportingClientCreation = errors.New("could not create reporting client") + ErrExportClientCreation = errors.New("could not create exporting client") ) // GetMatch returns true if a filter matches result event @@ -73,8 +49,8 @@ func isTagMatch(event *output.ResultEvent, filter *Filter) bool { } tags := event.Info.Tags.ToSlice() - for _, tag := range filterTags.ToSlice() { - if stringSliceContains(tags, tag) { + for _, filterTag := range filterTags.ToSlice() { + if sliceutil.Contains(tags, filterTag) { return true } } @@ -89,13 +65,7 @@ func isSeverityMatch(event *output.ResultEvent, filter *Filter) bool { return true } - for _, current := range filter.Severities { - if current == resultEventSeverity { - return true - } - } - - return false + return sliceutil.Contains(filter.Severities, resultEventSeverity) } // Tracker is an interface implemented by an issue tracker @@ -112,8 +82,8 @@ type Exporter interface { Export(event *output.ResultEvent) error } -// Client is a client for nuclei issue tracking module -type Client struct { +// ReportingClient is a client for nuclei issue tracking module +type ReportingClient struct { trackers []Tracker exporters []Exporter options *Options @@ -121,14 +91,14 @@ type Client struct { } // New creates a new nuclei issue tracker reporting client -func New(options *Options, db string) (*Client, error) { - client := &Client{options: options} +func New(options *Options, db string) (Client, error) { + client := &ReportingClient{options: options} if options.GitHub != nil { options.GitHub.HttpClient = options.HttpClient tracker, err := github.New(options.GitHub) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errors.Join(err, ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } @@ -136,7 +106,7 @@ func New(options *Options, db string) (*Client, error) { options.GitLab.HttpClient = options.HttpClient tracker, err := gitlab.New(options.GitLab) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errors.Join(err, ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } @@ -144,21 +114,21 @@ func New(options *Options, db string) (*Client, error) { options.Jira.HttpClient = options.HttpClient tracker, err := jira.New(options.Jira) if err != nil { - return nil, errors.Wrap(err, reportingClientCreationErrorMessage) + return nil, errors.Join(err, ErrReportingClientCreation) } client.trackers = append(client.trackers, tracker) } if options.MarkdownExporter != nil { exporter, err := markdown.New(options.MarkdownExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errors.Join(err, ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } if options.SarifExporter != nil { exporter, err := sarif.New(options.SarifExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errors.Join(err, ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -166,7 +136,7 @@ func New(options *Options, db string) (*Client, error) { options.ElasticsearchExporter.HttpClient = options.HttpClient exporter, err := es.New(options.ElasticsearchExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errors.Join(err, ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -174,7 +144,7 @@ func New(options *Options, db string) (*Client, error) { options.SplunkExporter.HttpClient = options.HttpClient exporter, err := splunk.New(options.SplunkExporter) if err != nil { - return nil, errors.Wrap(err, exportClientCreationErrorMessage) + return nil, errors.Join(err, ErrExportClientCreation) } client.exporters = append(client.exporters, exporter) } @@ -191,7 +161,7 @@ func New(options *Options, db string) (*Client, error) { func CreateConfigIfNotExists() error { config, err := config.GetConfigDir() if err != nil { - return errors.Wrap(err, "could not get config directory") + return errorutil.NewWithErr(err).Msgf("could not get config directory") } reportingConfig := filepath.Join(config, "report-config.yaml") @@ -213,7 +183,7 @@ func CreateConfigIfNotExists() error { } reportingFile, err := os.Create(reportingConfig) if err != nil { - return errors.Wrap(err, "could not create config file") + return errorutil.NewWithErr(err).Msgf("could not create config file") } defer reportingFile.Close() @@ -222,17 +192,17 @@ func CreateConfigIfNotExists() error { } // RegisterTracker registers a custom tracker to the reporter -func (c *Client) RegisterTracker(tracker Tracker) { +func (c *ReportingClient) RegisterTracker(tracker Tracker) { c.trackers = append(c.trackers, tracker) } // RegisterExporter registers a custom exporter to the reporter -func (c *Client) RegisterExporter(exporter Exporter) { +func (c *ReportingClient) RegisterExporter(exporter Exporter) { c.exporters = append(c.exporters, exporter) } // Close closes the issue tracker reporting client -func (c *Client) Close() { +func (c *ReportingClient) Close() { c.dedupe.Close() for _, exporter := range c.exporters { exporter.Close() @@ -240,7 +210,7 @@ func (c *Client) Close() { } // CreateIssue creates an issue in the tracker -func (c *Client) CreateIssue(event *output.ResultEvent) error { +func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error { if c.options.AllowList != nil && !c.options.AllowList.GetMatch(event) { return nil } @@ -252,27 +222,22 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error { if unique { for _, tracker := range c.trackers { if trackerErr := tracker.CreateIssue(event); trackerErr != nil { - err = multierr.Append(err, trackerErr) + err = errors.Join(err, trackerErr) } } for _, exporter := range c.exporters { if exportErr := exporter.Export(event); exportErr != nil { - err = multierr.Append(err, exportErr) + err = errors.Join(err, exportErr) } } } return err } -func stringSliceContains(slice []string, item string) bool { - for _, i := range slice { - if strings.EqualFold(i, item) { - return true - } - } - return false -} - -func (c *Client) GetReportingOptions() *Options { +func (c *ReportingClient) GetReportingOptions() *Options { return c.options } + +func (c *ReportingClient) Clear() { + c.dedupe.Clear() +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 95bed41e7..e079c9aef 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -3,6 +3,7 @@ package templates import ( "encoding/json" + "errors" validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -18,7 +19,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" - "go.uber.org/multierr" "gopkg.in/yaml.v2" ) @@ -154,7 +154,7 @@ func (template *Template) Type() types.ProtocolType { func (template *Template) MarshalYAML() ([]byte, error) { out, marshalErr := yaml.Marshal(template) errValidate := validate.New().Struct(template) - return out, multierr.Append(marshalErr, errValidate) + return out, errors.Join(marshalErr, errValidate) } // MarshalYAML forces recursive struct validation after unmarshal operation @@ -173,7 +173,7 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error func (template *Template) MarshalJSON() ([]byte, error) { out, marshalErr := json.Marshal(template) errValidate := validate.New().Struct(template) - return out, multierr.Append(marshalErr, errValidate) + return out, errors.Join(marshalErr, errValidate) } // UnmarshalJSON forces recursive struct validation after unmarshal operation