diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 9acf01951..b1764effd 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -77,7 +77,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode") set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID") set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file") - set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database") + set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)") set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for") set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags") set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") @@ -87,6 +87,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") + set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") _ = set.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index ba44e52c8..972c641ef 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -26,7 +26,7 @@ require ( github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/collaborator v0.0.2 github.com/projectdiscovery/fastdialer v0.0.8 - github.com/projectdiscovery/goflags v0.0.3 + github.com/projectdiscovery/goflags v0.0.4 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/rawhttp v0.0.6 diff --git a/v2/go.sum b/v2/go.sum index e82c0e49e..44e195edf 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -201,8 +201,8 @@ github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5 github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= -github.com/projectdiscovery/goflags v0.0.3 h1:5s9qblVIP/dQt7Mr3PMvZLvekEyioOS5oZtZ6ncLQHA= -github.com/projectdiscovery/goflags v0.0.3/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY= +github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c423a8bef..9a40d88b2 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -19,13 +19,15 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" "github.com/rs/xid" "go.uber.org/atomic" "go.uber.org/ratelimit" + "gopkg.in/yaml.v2" ) // Runner is a client for running the enumeration process. @@ -39,7 +41,7 @@ type Runner struct { catalog *catalog.Catalog progress progress.Progress colorizer aurora.Aurora - issuesClient *issues.Client + issuesClient *reporting.Client severityColors *colorizer.Colorizer browser *engine.Browser ratelimiter ratelimit.Limiter @@ -66,13 +68,36 @@ func New(options *types.Options) (*Runner, error) { } runner.catalog = catalog.New(runner.options.TemplatesDirectory) + var reportingOptions *reporting.Options if options.ReportingConfig != "" { - if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil { + file, err := os.Open(options.ReportingConfig) + if err != nil { + gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err) + } + + reportingOptions = &reporting.Options{} + if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil { + file.Close() + gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr) + } + file.Close() + } + if options.DiskExportDirectory != "" { + if reportingOptions != nil { + reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + } else { + reportingOptions = &reporting.Options{} + reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + } + } + if reportingOptions != nil { + if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil { gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) } else { runner.issuesClient = client } } + // output coloring useColor := !options.NoColor runner.colorizer = aurora.NewAurora(useColor) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 41c44a202..d4e89a02d 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -9,7 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" ) @@ -39,7 +39,7 @@ type ExecuterOptions struct { // Options contains configuration options for the executer. Options *types.Options // IssuesClient is a client for nuclei issue tracker reporting - IssuesClient *issues.Client + IssuesClient *reporting.Client // Progress is a progress client for scan reporting Progress progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. diff --git a/v2/pkg/reporting/issues/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go similarity index 100% rename from v2/pkg/reporting/issues/dedupe/dedupe.go rename to v2/pkg/reporting/dedupe/dedupe.go diff --git a/v2/pkg/reporting/issues/dedupe/dedupe_test.go b/v2/pkg/reporting/dedupe/dedupe_test.go similarity index 100% rename from v2/pkg/reporting/issues/dedupe/dedupe_test.go rename to v2/pkg/reporting/dedupe/dedupe_test.go diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go new file mode 100644 index 000000000..21f4a484e --- /dev/null +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -0,0 +1,91 @@ +package disk + +import ( + "bytes" + "crypto/sha1" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path" + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/segmentio/ksuid" +) + +type Exporter struct { + directory string + options *Options +} + +// Options contains the configuration options for github issue tracker client +type Options struct { + // Directory is the directory to export found results to + Directory string `yaml:"directory"` +} + +// New creates a new disk exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + directory := options.Directory + if options.Directory == "" { + dir, err := os.Getwd() + if err != nil { + return nil, err + } + directory = dir + } + _ = os.MkdirAll(directory, os.ModePerm) + return &Exporter{options: options, directory: directory}, nil +} + +// Export exports a passed result event to disk +func (i *Exporter) Export(event *output.ResultEvent) error { + summary := format.Summary(event) + description := format.MarkdownDescription(event) + + var filename string + if outputFile := baseFilenameFromURL(event.Matched, event.Type); outputFile != "" { + filename = outputFile + } else { + filename = ksuid.New().String() + } + + filenameBuilder := &strings.Builder{} + filenameBuilder.WriteString(filename) + filenameBuilder.WriteString(".md") + finalFilename := filenameBuilder.String() + + dataBuilder := &bytes.Buffer{} + dataBuilder.WriteString("### ") + dataBuilder.WriteString(summary) + dataBuilder.WriteString("\n---\n") + dataBuilder.WriteString(description) + data := dataBuilder.Bytes() + + err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644) + return err +} + +// Taken from https://github.com/michenriksen/aquatone/blob/854a5d56fbb7a00b2e5ec80d443026c7a4ced798/core/session.go#L215 +func baseFilenameFromURL(stru, protocol string) string { + u, err := url.Parse(stru) + if err != nil { + return "" + } + + h := sha1.New() + _, _ = io.WriteString(h, u.Path) + _, _ = io.WriteString(h, u.RawQuery) + _, _ = io.WriteString(h, u.Fragment) + + pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16] + host := strings.Replace(u.Host, ":", "__", 1) + if u.Scheme == "" { + u.Scheme = protocol + } + filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.ReplaceAll(host, ".", "_"), pathHash) + return strings.ToLower(filename) +} diff --git a/v2/pkg/reporting/issues/format/format.go b/v2/pkg/reporting/format/format.go similarity index 84% rename from v2/pkg/reporting/issues/format/format.go rename to v2/pkg/reporting/format/format.go index 1544ad519..1a823c3a6 100644 --- a/v2/pkg/reporting/issues/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -46,11 +46,21 @@ func MarkdownDescription(event *output.ResultEvent) string { for k, v := range event.Info { builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } - builder.WriteString("\n**Request**\n\n```\n") - builder.WriteString(event.Request) - builder.WriteString("\n```\n\n
**Response**\n\n```\n") - builder.WriteString(event.Response) - builder.WriteString("\n```\n\n") + if event.Request != "" { + builder.WriteString("\n**Request**\n\n```\n") + builder.WriteString(event.Request) + } + if event.Response != "" { + builder.WriteString("\n```\n\n**Response**\n\n```\n") + // If the response is larger than 5 kb, truncate it before writing. + if len(event.Response) > 5*1024 { + builder.WriteString(event.Response[:5*1024]) + builder.WriteString(".... Truncated ....") + } else { + builder.WriteString(event.Response) + } + builder.WriteString("\n```\n\n") + } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { builder.WriteString("**Extra Information**\n\n") diff --git a/v2/pkg/reporting/issues/issues.go b/v2/pkg/reporting/reporting.go similarity index 59% rename from v2/pkg/reporting/issues/issues.go rename to v2/pkg/reporting/reporting.go index 66f781504..e33600bb0 100644 --- a/v2/pkg/reporting/issues/issues.go +++ b/v2/pkg/reporting/reporting.go @@ -1,17 +1,17 @@ -package issues +package reporting import ( - "os" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "gopkg.in/yaml.v2" + "go.uber.org/multierr" ) // Options is a configuration file for nuclei reporting module @@ -26,6 +26,8 @@ type Options struct { Gitlab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` + // DiskExporter contains configuration options for Disk Exporter Module + DiskExporter *disk.Options `yaml:"disk"` } // Filter filters the received event and decides whether to perform @@ -75,25 +77,22 @@ type Tracker interface { CreateIssue(event *output.ResultEvent) error } +// Exporter is an interface implemented by an issue exporter +type Exporter interface { + // Export exports an issue to an exporter + Export(event *output.ResultEvent) error +} + // Client is a client for nuclei issue tracking module type Client struct { - tracker Tracker - options *Options - dedupe *dedupe.Storage + trackers []Tracker + exporters []Exporter + options *Options + dedupe *dedupe.Storage } // New creates a new nuclei issue tracker reporting client -func New(config, db string) (*Client, error) { - file, err := os.Open(config) - if err != nil { - return nil, errors.Wrap(err, "could not open reporting config file") - } - defer file.Close() - - options := &Options{} - if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil { - return nil, parseErr - } +func New(options *Options, db string) (*Client, error) { if options.AllowList != nil { options.AllowList.Compile() } @@ -101,27 +100,41 @@ func New(config, db string) (*Client, error) { options.DenyList.Compile() } - var tracker Tracker + client := &Client{options: options} if options.Github != nil { - tracker, err = github.New(options.Github) + tracker, err := github.New(options.Github) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } if options.Gitlab != nil { - tracker, err = gitlab.New(options.Gitlab) + tracker, err := gitlab.New(options.Gitlab) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } if options.Jira != nil { - tracker, err = jira.New(options.Jira) + tracker, err := jira.New(options.Jira) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } - if err != nil { - return nil, errors.Wrap(err, "could not create reporting client") - } - if tracker == nil { - return nil, errors.New("no issue tracker configuration found") + if options.DiskExporter != nil { + exporter, err := disk.New(options.DiskExporter) + if err != nil { + return nil, errors.Wrap(err, "could not create exporting client") + } + client.exporters = append(client.exporters, exporter) } storage, err := dedupe.New(db) if err != nil { return nil, err } - return &Client{tracker: tracker, dedupe: storage, options: options}, nil + client.dedupe = storage + return client, nil } // Close closes the issue tracker reporting client @@ -138,15 +151,20 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error { return nil } - found, err := c.dedupe.Index(event) - if err != nil { - _ = c.tracker.CreateIssue(event) - return err + unique, err := c.dedupe.Index(event) + if unique { + for _, tracker := range c.trackers { + if trackerErr := tracker.CreateIssue(event); trackerErr != nil { + err = multierr.Append(err, trackerErr) + } + } + for _, exporter := range c.exporters { + if exportErr := exporter.Export(event); exportErr != nil { + err = multierr.Append(err, exportErr) + } + } } - if found { - return c.tracker.CreateIssue(event) - } - return nil + return err } func stringSliceContains(slice []string, item string) bool { diff --git a/v2/pkg/reporting/issues/github/github.go b/v2/pkg/reporting/trackers/github/github.go similarity index 96% rename from v2/pkg/reporting/issues/github/github.go rename to v2/pkg/reporting/trackers/github/github.go index a74436347..89f6dceb7 100644 --- a/v2/pkg/reporting/issues/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-github/github" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" ) // Integration is a client for a issue tracker integration diff --git a/v2/pkg/reporting/issues/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go similarity index 96% rename from v2/pkg/reporting/issues/gitlab/gitlab.go rename to v2/pkg/reporting/trackers/gitlab/gitlab.go index 895507c85..33d4fff01 100644 --- a/v2/pkg/reporting/issues/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,7 +2,7 @@ package gitlab import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/xanzy/go-gitlab" ) diff --git a/v2/pkg/reporting/issues/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go similarity index 98% rename from v2/pkg/reporting/issues/jira/jira.go rename to v2/pkg/reporting/trackers/jira/jira.go index 0a0bfc80a..e0cf7eb38 100644 --- a/v2/pkg/reporting/issues/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -8,7 +8,7 @@ import ( jira "github.com/andygrunwald/go-jira" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dec4a3b65..53464684b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -43,6 +43,8 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string + // DiskExportDirectory is the directory to export reports in markdown on disk to + DiskExportDirectory string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after