mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-17 15:45:27 +00:00
MongoDB Reporting (#5688)
* Initial setup of Mongo reporting * Fix slice pop logic * Switch to config-file logic * Parse database name from connection string * Switch to url.Parse for connection string parsing * Address return/logging feedback
This commit is contained in:
Keith Chason
GitHub
parent
1cd42c46c7
commit
3f0de96726
@ -162,3 +162,14 @@
|
||||
# duplicate-issue-check: false
|
||||
# # open-state-id is the ID of the open state in Linear
|
||||
# open-state-id: ""
|
||||
#mongodb:
|
||||
# # the connection string to the MongoDB database
|
||||
# # (e.g., mongodb://root:example@localhost:27017/nuclei?ssl=false&authSource=admin)
|
||||
# connection-string: ""
|
||||
# # the name of the collection to store the issues
|
||||
# collection-name: ""
|
||||
# # excludes the Request and Response from the results (helps with filesize)
|
||||
# omit-raw: false
|
||||
# # determines the number of results to be kept in memory before writing it to the database or 0 to
|
||||
# # persist all in memory and write all results at the end (default)
|
||||
# batch-size: 0
|
||||
6
go.mod
6
go.mod
@ -105,6 +105,7 @@ require (
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
|
||||
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
|
||||
go.mongodb.org/mongo-driver v1.17.0
|
||||
golang.org/x/term v0.24.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
moul.io/http2curl v1.0.0
|
||||
@ -195,6 +196,7 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/moby/term v0.5.0 // indirect
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/muesli/reflow v0.3.0 // indirect
|
||||
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
@ -228,9 +230,13 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
github.com/ysmood/fetchup v0.2.3 // indirect
|
||||
github.com/ysmood/got v0.34.1 // indirect
|
||||
github.com/yuin/goldmark v1.7.4 // indirect
|
||||
|
||||
12
go.sum
12
go.sum
@ -755,6 +755,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
|
||||
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
|
||||
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
|
||||
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
|
||||
github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
|
||||
github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
|
||||
@ -1085,6 +1087,12 @@ github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd
|
||||
github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@ -1098,6 +1106,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
|
||||
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
|
||||
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
@ -1150,6 +1160,8 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
|
||||
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
|
||||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
|
||||
go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
|
||||
155
pkg/reporting/exporters/mongo/mongo.go
Normal file
155
pkg/reporting/exporters/mongo/mongo.go
Normal file
@ -0,0 +1,155 @@
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
mongooptions "go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type Exporter struct {
|
||||
options *Options
|
||||
mutex *sync.Mutex
|
||||
rows []output.ResultEvent
|
||||
collection *mongo.Collection
|
||||
connection *mongo.Client
|
||||
}
|
||||
|
||||
// Options contains the configuration options for MongoDB exporter client
|
||||
type Options struct {
|
||||
// ConnectionString is the connection string to the MongoDB database
|
||||
ConnectionString string `yaml:"connection-string"`
|
||||
// CollectionName is the name of the MongoDB collection in which to store the results
|
||||
CollectionName string `yaml:"collection-name"`
|
||||
// OmitRaw excludes the Request and Response from the results (helps with filesize)
|
||||
OmitRaw bool `yaml:"omit-raw"`
|
||||
// BatchSize determines the number of results to be kept in memory before writing it to the database or 0 to
|
||||
// persist all in memory and write all results at the end (default)
|
||||
BatchSize int `yaml:"batch-size"`
|
||||
}
|
||||
|
||||
// New creates a new MongoDB exporter integration client based on options.
|
||||
func New(options *Options) (*Exporter, error) {
|
||||
exporter := &Exporter{
|
||||
mutex: &sync.Mutex{},
|
||||
options: options,
|
||||
rows: []output.ResultEvent{},
|
||||
}
|
||||
|
||||
// If the environment variable for the connection string is set, then use that instead. This allows for easier
|
||||
// management of sensitive items such as credentials
|
||||
envConnectionString := os.Getenv("MONGO_CONNECTION_STRING")
|
||||
if envConnectionString != "" {
|
||||
options.ConnectionString = envConnectionString
|
||||
gologger.Info().Msgf("Using connection string from environment variable MONGO_CONNECTION_STRING")
|
||||
}
|
||||
|
||||
// Create the connection to the database
|
||||
clientOptions := mongooptions.Client().ApplyURI(options.ConnectionString)
|
||||
|
||||
// Create a new client and connect to the MongoDB server
|
||||
client, err := mongo.Connect(context.TODO(), clientOptions)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Error creating MongoDB client: %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Ensure the connection is valid
|
||||
err = client.Ping(context.Background(), nil)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Error connecting to MongoDB: %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get the database from the connection string to set the database and collection
|
||||
parsed, err := url.Parse(options.ConnectionString)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Error parsing connection string: %s", options.ConnectionString)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
databaseName := strings.TrimPrefix(parsed.Path, "/")
|
||||
|
||||
if databaseName == "" {
|
||||
return nil, errors.New("error getting database name from connection string")
|
||||
}
|
||||
|
||||
exporter.connection = client
|
||||
exporter.collection = client.Database(databaseName).Collection(options.CollectionName)
|
||||
|
||||
return exporter, nil
|
||||
}
|
||||
|
||||
// Export writes a result document to the configured MongoDB collection
|
||||
// in the database configured by the connection string
|
||||
func (exporter *Exporter) Export(event *output.ResultEvent) error {
|
||||
exporter.mutex.Lock()
|
||||
defer exporter.mutex.Unlock()
|
||||
|
||||
if exporter.options.OmitRaw {
|
||||
event.Request = ""
|
||||
event.Response = ""
|
||||
}
|
||||
|
||||
// Add the row to the queue to be processed
|
||||
exporter.rows = append(exporter.rows, *event)
|
||||
|
||||
// If the batch size is greater than 0 and the number of rows has reached the batch, flush it to the database
|
||||
if exporter.options.BatchSize > 0 && len(exporter.rows) >= exporter.options.BatchSize {
|
||||
err := exporter.WriteRows()
|
||||
if err != nil {
|
||||
// The error is already logged, return it to bubble up to the caller
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteRows writes all rows from the rows list to the MongoDB collection and removes them from the list
|
||||
func (exporter *Exporter) WriteRows() error {
|
||||
// Loop through the rows and write them, removing them as they're entered
|
||||
for len(exporter.rows) > 0 {
|
||||
data := exporter.rows[0]
|
||||
|
||||
// Write the data to the database
|
||||
_, err := exporter.collection.InsertOne(context.TODO(), data)
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Error inserting record into MongoDB collection: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove the item from the list
|
||||
exporter.rows = exporter.rows[1:]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (exporter *Exporter) Close() error {
|
||||
exporter.mutex.Lock()
|
||||
defer exporter.mutex.Unlock()
|
||||
|
||||
// Write all pending rows
|
||||
err := exporter.WriteRows()
|
||||
if err != nil {
|
||||
// The error is already logged, return it to bubble up to the caller
|
||||
return err
|
||||
}
|
||||
|
||||
// Close the database connection
|
||||
err = exporter.connection.Disconnect(context.TODO())
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Error disconnecting from MongoDB: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
|
||||
@ -44,6 +45,8 @@ type Options struct {
|
||||
JSONExporter *jsonexporter.Options `yaml:"json"`
|
||||
// JSONLExporter contains configuration options for JSONL Exporter Module
|
||||
JSONLExporter *jsonl.Options `yaml:"jsonl"`
|
||||
// MongoDBExporter containers the configuration options for the MongoDB Exporter Module
|
||||
MongoDBExporter *mongo.Options `yaml:"mongodb"`
|
||||
|
||||
HttpClient *retryablehttp.Client `yaml:"-"`
|
||||
OmitRaw bool `yaml:"-"`
|
||||
|
||||
@ -2,6 +2,7 @@ package reporting
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@ -166,6 +167,13 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) {
|
||||
}
|
||||
client.exporters = append(client.exporters, exporter)
|
||||
}
|
||||
if options.MongoDBExporter != nil {
|
||||
exporter, err := mongo.New(options.MongoDBExporter)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation)
|
||||
}
|
||||
client.exporters = append(client.exporters, exporter)
|
||||
}
|
||||
|
||||
if doNotDedupe {
|
||||
return client, nil
|
||||
@ -212,6 +220,7 @@ func CreateConfigIfNotExists() error {
|
||||
SplunkExporter: &splunk.Options{},
|
||||
JSONExporter: &json_exporter.Options{},
|
||||
JSONLExporter: &jsonl.Options{},
|
||||
MongoDBExporter: &mongo.Options{},
|
||||
}
|
||||
reportingFile, err := os.Create(reportingConfig)
|
||||
if err != nil {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user