External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:35:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

omit raw from integrations (#4612)

Browse Source 
* omit raw from integrations

* fix lint
This commit is contained in:
Dogan Can Bakir 2024-01-27 01:36:25 +03:00 committed by GitHub
parent b9e2665e9e
commit e102caec78
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 57 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/runner/options.go
View File

@ -262,9 +262,9 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
if options.MarkdownExportDirectory != "" {
reportingOptions.MarkdownExporter = &markdown.Options{
Directory: options.MarkdownExportDirectory,
IncludeRawPayload: !options.OmitRawRequests,
SortMode: options.MarkdownExportSortMode,
Directory: options.MarkdownExportDirectory,
OmitRaw: options.OmitRawRequests,
SortMode: options.MarkdownExportSortMode,
}
}
if options.SarifExport != "" {
@ -272,17 +272,18 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
if options.JSONExport != "" {
reportingOptions.JSONExporter = &jsonexporter.Options{
File: options.JSONExport,
IncludeRawPayload: !options.OmitRawRequests,
File: options.JSONExport,
OmitRaw: options.OmitRawRequests,
}
}
if options.JSONLExport != "" {
reportingOptions.JSONLExporter = &jsonl.Options{
File: options.JSONLExport,
IncludeRawPayload: !options.OmitRawRequests,
File: options.JSONLExport,
OmitRaw: options.OmitRawRequests,
}
}
reportingOptions.OmitRaw = options.OmitRawRequests
return reportingOptions, nil
}

15
pkg/reporting/exporters/jsonexporter/jsonexporter.go
View File

@ -2,10 +2,11 @@ package jsonexporter
import (
"encoding/json"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"os"
"sync"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
type Exporter struct {
@ -17,8 +18,8 @@ type Exporter struct {
// Options contains the configuration options for JSON exporter client
type Options struct {
// File is the file to export found JSON result to
File string `yaml:"file"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
File string `yaml:"file"`
OmitRaw bool `yaml:"omit-raw"`
}
// New creates a new JSON exporter integration client based on options.
@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
if exporter.options.OmitRaw {
event.Request = ""
event.Response = ""
}

15
pkg/reporting/exporters/jsonl/jsonl.go
View File

@ -2,10 +2,11 @@ package jsonl
import (
"encoding/json"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"os"
"sync"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
type Exporter struct {
@ -17,8 +18,8 @@ type Exporter struct {
// Options contains the configuration options for JSONL exporter client
type Options struct {
// File is the file to export found JSONL result to
File string `yaml:"file"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
File string `yaml:"file"`
OmitRaw bool `yaml:"omit-raw"`
}
// New creates a new JSONL exporter integration client based on options.
@ -37,11 +38,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
if exporter.options.OmitRaw {
event.Request = ""
event.Response = ""
}

17
pkg/reporting/exporters/markdown/markdown.go
View File

@ -26,9 +26,9 @@ type Exporter struct {
// Options contains the configuration options for GitHub issue tracker client
type Options struct {
// Directory is the directory to export found results to
Directory string `yaml:"directory"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
SortMode string `yaml:"sort-mode"`
Directory string `yaml:"directory"`
OmitRaw bool `yaml:"omit-raw"`
SortMode string `yaml:"sort-mode"`
}
// New creates a new markdown exporter integration client based on options.
@ -56,15 +56,6 @@ func New(options *Options) (*Exporter, error) {
// Export exports a passed result event to markdown
func (exporter *Exporter) Export(event *output.ResultEvent) error {
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the markdown report output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
event.Request = ""
event.Response = ""
}
// index file generation
file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
@ -114,7 +105,7 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
dataBuilder.WriteString(util.CreateHeading3(format.Summary(event)))
dataBuilder.WriteString("\n")
dataBuilder.WriteString(util.CreateHorizontalLine())
dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{}))
dataBuilder.WriteString(format.CreateReportDescription(event, util.MarkdownFormatter{}, exporter.options.OmitRaw))
data := dataBuilder.Bytes()
return os.WriteFile(filepath.Join(exporter.directory, subdirectory, filename), data, 0644)

30
pkg/reporting/format/format_utils.go
View File

@ -34,7 +34,7 @@ func GetMatchedTemplateName(event *output.ResultEvent) string {
return matchedTemplateName
}
func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter) string {
func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatter, omitRaw bool) string {
template := GetMatchedTemplateName(event)
builder := &bytes.Buffer{}
builder.WriteString(fmt.Sprintf("%s: %s matched at %s\n\n", formatter.MakeBold("Details"), formatter.MakeBold(template), event.Host))
@ -51,20 +51,22 @@ func CreateReportDescription(event *output.ResultEvent, formatter ResultFormatte
builder.WriteString("\n\n")
builder.WriteString(CreateTemplateInfoTable(&event.Info, formatter))
if event.Request != "" {
builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http"))
}
if event.Response != "" {
var responseString string
// If the response is larger than 5 kb, truncate it before writing.
maxKbSize := 5 * 1024
if len(event.Response) > maxKbSize {
responseString = event.Response[:maxKbSize]
responseString += ".... Truncated ...."
} else {
responseString = event.Response
if !omitRaw {
if event.Request != "" {
builder.WriteString(formatter.CreateCodeBlock("Request", types.ToHexOrString(event.Request), "http"))
}
if event.Response != "" {
var responseString string
// If the response is larger than 5 kb, truncate it before writing.
maxKbSize := 5 * 1024
if len(event.Response) > maxKbSize {
responseString = event.Response[:maxKbSize]
responseString += ".... Truncated ...."
} else {
responseString = event.Response
}
builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http"))
}
builder.WriteString(formatter.CreateCodeBlock("Response", responseString, "http"))
}
if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {

1
pkg/reporting/options.go
View File

@ -39,4 +39,5 @@ type Options struct {
JSONLExporter *jsonl.Options `yaml:"jsonl"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
}

3
pkg/reporting/reporting.go
View File

@ -99,6 +99,7 @@ func New(options *Options, db string) (Client, error) {
if options.GitHub != nil {
options.GitHub.HttpClient = options.HttpClient
options.GitHub.OmitRaw = options.OmitRaw
tracker, err := github.New(options.GitHub)
if err != nil {
return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
@ -107,6 +108,7 @@ func New(options *Options, db string) (Client, error) {
}
if options.GitLab != nil {
options.GitLab.HttpClient = options.HttpClient
options.GitLab.OmitRaw = options.OmitRaw
tracker, err := gitlab.New(options.GitLab)
if err != nil {
return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)
@ -115,6 +117,7 @@ func New(options *Options, db string) (Client, error) {
}
if options.Jira != nil {
options.Jira.HttpClient = options.HttpClient
options.Jira.OmitRaw = options.OmitRaw
tracker, err := jira.New(options.Jira)
if err != nil {
return nil, errorutil.NewWithErr(err).Wrap(ErrReportingClientCreation)

12
pkg/reporting/trackers/github/github.go
View File

@ -3,6 +3,11 @@ package github
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"github.com/google/go-github/github"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
@ -11,10 +16,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/oauth2"
"io"
"net/http"
"net/url"
"strings"
)
// Integration is a client for an issue tracker integration
@ -45,6 +46,7 @@ type Options struct {
DuplicateIssueCheck bool `yaml:"duplicate-issue-check"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
}
// New creates a new issue tracker integration client based on options.
@ -80,7 +82,7 @@ func New(options *Options) (*Integration, error) {
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) (err error) {
summary := format.Summary(event)
description := format.CreateReportDescription(event, util.MarkdownFormatter{})
description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)
labels := []string{}
severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
if i.options.SeverityAsLabel && severityLabel != "" {

3
pkg/reporting/trackers/gitlab/gitlab.go
View File

@ -37,6 +37,7 @@ type Options struct {
DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
}
// New creates a new issue tracker integration client based on options.
@ -62,7 +63,7 @@ func New(options *Options) (*Integration, error) {
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)
description := format.CreateReportDescription(event, util.MarkdownFormatter{})
description := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)
labels := []string{}
severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String())
if i.options.SeverityAsLabel && severityLabel != "" {

7
pkg/reporting/trackers/jira/jira.go
View File

@ -77,6 +77,7 @@ type Options struct {
// that will be used to create the issue
CustomFields map[string]interface{} `yaml:"custom-fields" json:"custom_fields"`
StatusNot string `yaml:"status-not" json:"status_not"`
OmitRaw bool `yaml:"-"`
}
// New creates a new issue tracker integration client based on options.
@ -154,7 +155,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
}
}
fields := &jira.IssueFields{
Description: format.CreateReportDescription(event, i),
Description: format.CreateReportDescription(event, i, i.options.OmitRaw),
Unknowns: customFields,
Type: jira.IssueType{Name: i.options.IssueType},
Project: jira.Project{Key: i.options.ProjectName},
@ -164,7 +165,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
if !i.options.Cloud {
fields = &jira.IssueFields{
Assignee: &jira.User{Name: i.options.AccountID},
Description: format.CreateReportDescription(event, i),
Description: format.CreateReportDescription(event, i, i.options.OmitRaw),
Type: jira.IssueType{Name: i.options.IssueType},
Project: jira.Project{Key: i.options.ProjectName},
Summary: summary,
@ -196,7 +197,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error {
return err
} else if issueID != "" {
_, _, err = i.jira.Issue.AddComment(issueID, &jira.Comment{
Body: format.CreateReportDescription(event, i),
Body: format.CreateReportDescription(event, i, i.options.OmitRaw),
})
return err
}