External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 18:45:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

converting reporting client to interface

Browse Source
This commit is contained in:
Mzack9999 2023-02-07 09:45:49 +01:00
parent 1e5358b1fa
commit d57aec5ec7
11 changed files with 119 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/go.mod
View File

@ -43,7 +43,7 @@ require (
github.com/valyala/fasttemplate v1.2.2
github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37
github.com/xanzy/go-gitlab v0.79.0
go.uber.org/multierr v1.9.0
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/net v0.5.0
golang.org/x/oauth2 v0.4.0
golang.org/x/text v0.6.0

2
v2/internal/runner/runner.go
View File

@ -70,7 +70,7 @@ type Runner struct {
catalog catalog.Catalog
progress progress.Progress
colorizer aurora.Aurora
issuesClient *reporting.Client
issuesClient reporting.Client
hmapInputProvider *hybrid.Input
browser *engine.Browser
ratelimiter *ratelimit.Limiter

2
v2/pkg/protocols/common/helpers/writer/writer.go
View File

@ -8,7 +8,7 @@ import (
)
// WriteResult is a helper for writing results to the output
func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool {
func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient reporting.Client) bool {
// Handle the case where no result found for the template.
// In this case, we just show misc information about the failed
// match for the template.

4
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -84,7 +84,7 @@ type Options struct {
// Output is the output writer for nuclei
Output output.Writer
// IssuesClient is a client for issue exporting
IssuesClient *reporting.Client
IssuesClient reporting.Client
// Progress is the nuclei progress bar implementation.
Progress progress.Progress
// Debug specifies whether debugging output should be shown for interactsh-client
@ -132,7 +132,7 @@ func New(options *Options) (*Client, error) {
}
// NewDefaultOptions returns the default options for interactsh client
func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options {
func NewDefaultOptions(output output.Writer, reporting reporting.Client, progress progress.Progress) *Options {
return &Options{
ServerURL: client.DefaultOptions.ServerURL,
CacheSize: 5000,

25
v2/pkg/protocols/http/request.go
View File

@ -13,9 +13,9 @@ import (
"sync"
"time"
"github.com/pkg/errors"
"errors"
"github.com/remeh/sizedwaitgroup"
"go.uber.org/multierr"
"moul.io/http2curl"
"github.com/projectdiscovery/gologger"
@ -36,6 +36,7 @@ import (
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/rawhttp"
errorutil "github.com/projectdiscovery/utils/errors"
stringsutil "github.com/projectdiscovery/utils/strings"
urlutil "github.com/projectdiscovery/utils/url"
)
@ -107,7 +108,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
requestErr = errors.Join(requestErr, err)
}
mutex.Unlock()
}(generatedRequests[i])
@ -154,7 +155,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
requestErr = errors.Join(requestErr, err)
}
mutex.Unlock()
}(generatedHttpRequest)
@ -217,7 +218,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
requestErr = errors.Join(requestErr, err)
}
mutex.Unlock()
}(generatedHttpRequest)
@ -231,7 +232,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
parsed, err := urlutil.Parse(input.MetaInput.Input)
if err != nil {
return errors.Wrap(err, "could not parse url")
return errorutil.NewWithErr(err).Msgf("could not parse url")
}
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
@ -304,7 +305,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
return nil
}
if err != nil {
return errors.Wrap(err, "could not execute rule")
return errorutil.NewWithErr(err).Msgf("could not execute rule")
}
}
}
@ -558,7 +559,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
connConfiguration.Connection.Cookiejar = input.CookieJar
client, err := httpclientpool.Get(request.options.Options, connConfiguration)
if err != nil {
return errors.Wrap(err, "could not get http client")
return errorutil.NewWithErr(err).Msgf("could not get http client")
}
httpclient = client
}
@ -645,7 +646,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
dumpedResponseHeaders, err := httputil.DumpResponse(resp, false)
if err != nil {
return errors.Wrap(err, "could not dump http response")
return errorutil.NewWithErr(err).Msgf("could not dump http response")
}
var dumpedResponse []redirectedResponse
@ -666,7 +667,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error())
} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error
return errors.Wrap(err, "could not read http body")
return errorutil.NewWithErr(err).Msgf("could not read http body")
}
}
gotData = data
@ -674,7 +675,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
dumpedResponse, err = dumpResponseWithRedirectChain(resp, data)
if err != nil {
return errors.Wrap(err, "could not read http response with redirect chain")
return errorutil.NewWithErr(err).Msgf("could not read http response with redirect chain")
}
} else {
dumpedResponse = []redirectedResponse{{resp: resp, fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}}
@ -683,7 +684,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// if nuclei-project is enabled store the response if not previously done
if request.options.ProjectFile != nil && !fromCache {
if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil {
return errors.Wrap(err, "could not store in project file")
return errorutil.NewWithErr(err).Msgf("could not store in project file")
}
}

2
v2/pkg/protocols/protocols.go
View File

@ -50,7 +50,7 @@ type ExecuterOptions struct {
// Options contains configuration options for the executer.
Options *types.Options
// IssuesClient is a client for nuclei issue tracker reporting
IssuesClient *reporting.Client
IssuesClient reporting.Client
// Progress is a progress client for scan reporting
Progress progress.Progress
// RateLimiter is a rate-limiter for limiting sent number of requests.

15
v2/pkg/reporting/client.go Normal file
View File

@ -0,0 +1,15 @@
package reporting
import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
)
// Client is a client for nuclei issue tracking module
type Client interface {
RegisterTracker(tracker Tracker)
RegisterExporter(exporter Exporter)
Close()
Clear()
CreateIssue(event *output.ResultEvent) error
GetReportingOptions() *Options
}

12
v2/pkg/reporting/dedupe/dedupe.go
View File

@ -51,6 +51,18 @@ func New(dbPath string) (*Storage, error) {
return storage, nil
}
func (s *Storage) Clear() {
var keys [][]byte
iter := s.storage.NewIterator(nil, nil)
for iter.Next() {
keys = append(keys, iter.Key())
}
iter.Release()
for _, key := range keys {
s.storage.Delete(key, nil)
}
}
// Close closes the storage for further operations
func (s *Storage) Close() {
s.storage.Close()

36
v2/pkg/reporting/options.go Normal file
View File

@ -0,0 +1,36 @@
package reporting
import (
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/retryablehttp-go"
)
// Options is a configuration file for nuclei reporting module
type Options struct {
// AllowList contains a list of allowed events for reporting module
AllowList *Filter `yaml:"allow-list"`
// DenyList contains a list of denied events for reporting module
DenyList *Filter `yaml:"deny-list"`
// GitHub contains configuration options for GitHub Issue Tracker
GitHub *github.Options `yaml:"github"`
// GitLab contains configuration options for GitLab Issue Tracker
GitLab *gitlab.Options `yaml:"gitlab"`
// Jira contains configuration options for Jira Issue Tracker
Jira *jira.Options `yaml:"jira"`
// MarkdownExporter contains configuration options for Markdown Exporter Module
MarkdownExporter *markdown.Options `yaml:"markdown"`
// SarifExporter contains configuration options for Sarif Exporter Module
SarifExporter *sarif.Options `yaml:"sarif"`
// ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module
ElasticsearchExporter *es.Options `yaml:"elasticsearch"`
// SplunkExporter contains configuration options for splunkhec Exporter Module
SplunkExporter *splunk.Options `yaml:"splunkhec"`
HttpClient *retryablehttp.Client `yaml:"-"`
}

103
v2/pkg/reporting/reporting.go
View File

@ -3,12 +3,11 @@ package reporting
import (
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"go.uber.org/multierr"
"gopkg.in/yaml.v2"
"errors"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
@ -21,34 +20,11 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/retryablehttp-go"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
sliceutil "github.com/projectdiscovery/utils/slice"
)
// Options is a configuration file for nuclei reporting module
type Options struct {
// AllowList contains a list of allowed events for reporting module
AllowList *Filter `yaml:"allow-list"`
// DenyList contains a list of denied events for reporting module
DenyList *Filter `yaml:"deny-list"`
// GitHub contains configuration options for GitHub Issue Tracker
GitHub *github.Options `yaml:"github"`
// GitLab contains configuration options for GitLab Issue Tracker
GitLab *gitlab.Options `yaml:"gitlab"`
// Jira contains configuration options for Jira Issue Tracker
Jira *jira.Options `yaml:"jira"`
// MarkdownExporter contains configuration options for Markdown Exporter Module
MarkdownExporter *markdown.Options `yaml:"markdown"`
// SarifExporter contains configuration options for Sarif Exporter Module
SarifExporter *sarif.Options `yaml:"sarif"`
// ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module
ElasticsearchExporter *es.Options `yaml:"elasticsearch"`
// SplunkExporter contains configuration options for splunkhec Exporter Module
SplunkExporter *splunk.Options `yaml:"splunkhec"`
HttpClient *retryablehttp.Client `yaml:"-"`
}
// Filter filters the received event and decides whether to perform
// reporting for it or not.
type Filter struct {
@ -56,9 +32,9 @@ type Filter struct {
Tags stringslice.StringSlice `yaml:"tags"`
}
const (
reportingClientCreationErrorMessage = "could not create reporting client"
exportClientCreationErrorMessage = "could not create exporting client"
var (
ErrReportingClientCreation = errors.New("could not create reporting client")
ErrExportClientCreation = errors.New("could not create exporting client")
)
// GetMatch returns true if a filter matches result event
@ -73,8 +49,8 @@ func isTagMatch(event *output.ResultEvent, filter *Filter) bool {
}
tags := event.Info.Tags.ToSlice()
for _, tag := range filterTags.ToSlice() {
if stringSliceContains(tags, tag) {
for _, filterTag := range filterTags.ToSlice() {
if sliceutil.Contains(tags, filterTag) {
return true
}
}
@ -89,13 +65,7 @@ func isSeverityMatch(event *output.ResultEvent, filter *Filter) bool {
return true
}
for _, current := range filter.Severities {
if current == resultEventSeverity {
return true
}
}
return false
return sliceutil.Contains(filter.Severities, resultEventSeverity)
}
// Tracker is an interface implemented by an issue tracker
@ -112,8 +82,8 @@ type Exporter interface {
Export(event *output.ResultEvent) error
}
// Client is a client for nuclei issue tracking module
type Client struct {
// ReportingClient is a client for nuclei issue tracking module
type ReportingClient struct {
trackers []Tracker
exporters []Exporter
options *Options
@ -121,14 +91,14 @@ type Client struct {
}
// New creates a new nuclei issue tracker reporting client
func New(options *Options, db string) (*Client, error) {
client := &Client{options: options}
func New(options *Options, db string) (Client, error) {
client := &ReportingClient{options: options}
if options.GitHub != nil {
options.GitHub.HttpClient = options.HttpClient
tracker, err := github.New(options.GitHub)
if err != nil {
return nil, errors.Wrap(err, reportingClientCreationErrorMessage)
return nil, errors.Join(err, ErrReportingClientCreation)
}
client.trackers = append(client.trackers, tracker)
}
@ -136,7 +106,7 @@ func New(options *Options, db string) (*Client, error) {
options.GitLab.HttpClient = options.HttpClient
tracker, err := gitlab.New(options.GitLab)
if err != nil {
return nil, errors.Wrap(err, reportingClientCreationErrorMessage)
return nil, errors.Join(err, ErrReportingClientCreation)
}
client.trackers = append(client.trackers, tracker)
}
@ -144,21 +114,21 @@ func New(options *Options, db string) (*Client, error) {
options.Jira.HttpClient = options.HttpClient
tracker, err := jira.New(options.Jira)
if err != nil {
return nil, errors.Wrap(err, reportingClientCreationErrorMessage)
return nil, errors.Join(err, ErrReportingClientCreation)
}
client.trackers = append(client.trackers, tracker)
}
if options.MarkdownExporter != nil {
exporter, err := markdown.New(options.MarkdownExporter)
if err != nil {
return nil, errors.Wrap(err, exportClientCreationErrorMessage)
return nil, errors.Join(err, ErrExportClientCreation)
}
client.exporters = append(client.exporters, exporter)
}
if options.SarifExporter != nil {
exporter, err := sarif.New(options.SarifExporter)
if err != nil {
return nil, errors.Wrap(err, exportClientCreationErrorMessage)
return nil, errors.Join(err, ErrExportClientCreation)
}
client.exporters = append(client.exporters, exporter)
}
@ -166,7 +136,7 @@ func New(options *Options, db string) (*Client, error) {
options.ElasticsearchExporter.HttpClient = options.HttpClient
exporter, err := es.New(options.ElasticsearchExporter)
if err != nil {
return nil, errors.Wrap(err, exportClientCreationErrorMessage)
return nil, errors.Join(err, ErrExportClientCreation)
}
client.exporters = append(client.exporters, exporter)
}
@ -174,7 +144,7 @@ func New(options *Options, db string) (*Client, error) {
options.SplunkExporter.HttpClient = options.HttpClient
exporter, err := splunk.New(options.SplunkExporter)
if err != nil {
return nil, errors.Wrap(err, exportClientCreationErrorMessage)
return nil, errors.Join(err, ErrExportClientCreation)
}
client.exporters = append(client.exporters, exporter)
}
@ -191,7 +161,7 @@ func New(options *Options, db string) (*Client, error) {
func CreateConfigIfNotExists() error {
config, err := config.GetConfigDir()
if err != nil {
return errors.Wrap(err, "could not get config directory")
return errorutil.NewWithErr(err).Msgf("could not get config directory")
}
reportingConfig := filepath.Join(config, "report-config.yaml")
@ -213,7 +183,7 @@ func CreateConfigIfNotExists() error {
}
reportingFile, err := os.Create(reportingConfig)
if err != nil {
return errors.Wrap(err, "could not create config file")
return errorutil.NewWithErr(err).Msgf("could not create config file")
}
defer reportingFile.Close()
@ -222,17 +192,17 @@ func CreateConfigIfNotExists() error {
}
// RegisterTracker registers a custom tracker to the reporter
func (c *Client) RegisterTracker(tracker Tracker) {
func (c *ReportingClient) RegisterTracker(tracker Tracker) {
c.trackers = append(c.trackers, tracker)
}
// RegisterExporter registers a custom exporter to the reporter
func (c *Client) RegisterExporter(exporter Exporter) {
func (c *ReportingClient) RegisterExporter(exporter Exporter) {
c.exporters = append(c.exporters, exporter)
}
// Close closes the issue tracker reporting client
func (c *Client) Close() {
func (c *ReportingClient) Close() {
c.dedupe.Close()
for _, exporter := range c.exporters {
exporter.Close()
@ -240,7 +210,7 @@ func (c *Client) Close() {
}
// CreateIssue creates an issue in the tracker
func (c *Client) CreateIssue(event *output.ResultEvent) error {
func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
if c.options.AllowList != nil && !c.options.AllowList.GetMatch(event) {
return nil
}
@ -252,27 +222,22 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error {
if unique {
for _, tracker := range c.trackers {
if trackerErr := tracker.CreateIssue(event); trackerErr != nil {
err = multierr.Append(err, trackerErr)
err = errors.Join(err, trackerErr)
}
}
for _, exporter := range c.exporters {
if exportErr := exporter.Export(event); exportErr != nil {
err = multierr.Append(err, exportErr)
err = errors.Join(err, exportErr)
}
}
}
return err
}
func stringSliceContains(slice []string, item string) bool {
for _, i := range slice {
if strings.EqualFold(i, item) {
return true
}
}
return false
}
func (c *Client) GetReportingOptions() *Options {
func (c *ReportingClient) GetReportingOptions() *Options {
return c.options
}
func (c *ReportingClient) Clear() {
c.dedupe.Clear()
}

6
v2/pkg/templates/templates.go
View File

@ -3,6 +3,7 @@ package templates
import (
"encoding/json"
"errors"
validate "github.com/go-playground/validator/v10"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
@ -18,7 +19,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"go.uber.org/multierr"
"gopkg.in/yaml.v2"
)
@ -154,7 +154,7 @@ func (template *Template) Type() types.ProtocolType {
func (template *Template) MarshalYAML() ([]byte, error) {
out, marshalErr := yaml.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
return out, errors.Join(marshalErr, errValidate)
}
// MarshalYAML forces recursive struct validation after unmarshal operation
@ -173,7 +173,7 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
func (template *Template) MarshalJSON() ([]byte, error) {
out, marshalErr := json.Marshal(template)
errValidate := validate.New().Struct(template)
return out, multierr.Append(marshalErr, errValidate)
return out, errors.Join(marshalErr, errValidate)
}
// UnmarshalJSON forces recursive struct validation after unmarshal operation