diff --git a/cmd/nuclei/issue-tracker-config.yaml b/cmd/nuclei/issue-tracker-config.yaml index b7e0e6daf..07ccf828f 100644 --- a/cmd/nuclei/issue-tracker-config.yaml +++ b/cmd/nuclei/issue-tracker-config.yaml @@ -162,3 +162,14 @@ # duplicate-issue-check: false # # open-state-id is the ID of the open state in Linear # open-state-id: "" +#mongodb: +# # the connection string to the MongoDB database +# # (e.g., mongodb://root:example@localhost:27017/nuclei?ssl=false&authSource=admin) +# connection-string: "" +# # the name of the collection to store the issues +# collection-name: "" +# # excludes the Request and Response from the results (helps with filesize) +# omit-raw: false +# # determines the number of results to be kept in memory before writing it to the database or 0 to +# # persist all in memory and write all results at the end (default) +# batch-size: 0 \ No newline at end of file diff --git a/go.mod b/go.mod index f2adacca2..86a8a0064 100644 --- a/go.mod +++ b/go.mod @@ -105,6 +105,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 + go.mongodb.org/mongo-driver v1.17.0 golang.org/x/term v0.24.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 @@ -195,6 +196,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -228,9 +230,13 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/got v0.34.1 // indirect github.com/yuin/goldmark v1.7.4 // indirect diff --git a/go.sum b/go.sum index da71e9b28..b69d6283a 100644 --- a/go.sum +++ b/go.sum @@ -755,6 +755,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -1085,6 +1087,12 @@ github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1098,6 +1106,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= @@ -1150,6 +1160,8 @@ go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= +go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/pkg/reporting/exporters/mongo/mongo.go b/pkg/reporting/exporters/mongo/mongo.go new file mode 100644 index 000000000..faf8bb579 --- /dev/null +++ b/pkg/reporting/exporters/mongo/mongo.go @@ -0,0 +1,155 @@ +package mongo + +import ( + "context" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "go.mongodb.org/mongo-driver/mongo" + "net/url" + "os" + "strings" + "sync" + + mongooptions "go.mongodb.org/mongo-driver/mongo/options" +) + +type Exporter struct { + options *Options + mutex *sync.Mutex + rows []output.ResultEvent + collection *mongo.Collection + connection *mongo.Client +} + +// Options contains the configuration options for MongoDB exporter client +type Options struct { + // ConnectionString is the connection string to the MongoDB database + ConnectionString string `yaml:"connection-string"` + // CollectionName is the name of the MongoDB collection in which to store the results + CollectionName string `yaml:"collection-name"` + // OmitRaw excludes the Request and Response from the results (helps with filesize) + OmitRaw bool `yaml:"omit-raw"` + // BatchSize determines the number of results to be kept in memory before writing it to the database or 0 to + // persist all in memory and write all results at the end (default) + BatchSize int `yaml:"batch-size"` +} + +// New creates a new MongoDB exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + exporter := &Exporter{ + mutex: &sync.Mutex{}, + options: options, + rows: []output.ResultEvent{}, + } + + // If the environment variable for the connection string is set, then use that instead. This allows for easier + // management of sensitive items such as credentials + envConnectionString := os.Getenv("MONGO_CONNECTION_STRING") + if envConnectionString != "" { + options.ConnectionString = envConnectionString + gologger.Info().Msgf("Using connection string from environment variable MONGO_CONNECTION_STRING") + } + + // Create the connection to the database + clientOptions := mongooptions.Client().ApplyURI(options.ConnectionString) + + // Create a new client and connect to the MongoDB server + client, err := mongo.Connect(context.TODO(), clientOptions) + if err != nil { + gologger.Error().Msgf("Error creating MongoDB client: %s", err) + return nil, err + } + + // Ensure the connection is valid + err = client.Ping(context.Background(), nil) + if err != nil { + gologger.Error().Msgf("Error connecting to MongoDB: %s", err) + return nil, err + } + + // Get the database from the connection string to set the database and collection + parsed, err := url.Parse(options.ConnectionString) + if err != nil { + gologger.Error().Msgf("Error parsing connection string: %s", options.ConnectionString) + return nil, err + } + + databaseName := strings.TrimPrefix(parsed.Path, "/") + + if databaseName == "" { + return nil, errors.New("error getting database name from connection string") + } + + exporter.connection = client + exporter.collection = client.Database(databaseName).Collection(options.CollectionName) + + return exporter, nil +} + +// Export writes a result document to the configured MongoDB collection +// in the database configured by the connection string +func (exporter *Exporter) Export(event *output.ResultEvent) error { + exporter.mutex.Lock() + defer exporter.mutex.Unlock() + + if exporter.options.OmitRaw { + event.Request = "" + event.Response = "" + } + + // Add the row to the queue to be processed + exporter.rows = append(exporter.rows, *event) + + // If the batch size is greater than 0 and the number of rows has reached the batch, flush it to the database + if exporter.options.BatchSize > 0 && len(exporter.rows) >= exporter.options.BatchSize { + err := exporter.WriteRows() + if err != nil { + // The error is already logged, return it to bubble up to the caller + return err + } + } + + return nil +} + +// WriteRows writes all rows from the rows list to the MongoDB collection and removes them from the list +func (exporter *Exporter) WriteRows() error { + // Loop through the rows and write them, removing them as they're entered + for len(exporter.rows) > 0 { + data := exporter.rows[0] + + // Write the data to the database + _, err := exporter.collection.InsertOne(context.TODO(), data) + if err != nil { + gologger.Fatal().Msgf("Error inserting record into MongoDB collection: %s", err) + return err + } + + // Remove the item from the list + exporter.rows = exporter.rows[1:] + } + + return nil +} + +func (exporter *Exporter) Close() error { + exporter.mutex.Lock() + defer exporter.mutex.Unlock() + + // Write all pending rows + err := exporter.WriteRows() + if err != nil { + // The error is already logged, return it to bubble up to the caller + return err + } + + // Close the database connection + err = exporter.connection.Disconnect(context.TODO()) + if err != nil { + gologger.Error().Msgf("Error disconnecting from MongoDB: %s", err) + return err + } + + return nil +} diff --git a/pkg/reporting/options.go b/pkg/reporting/options.go index c5090de01..bda9b6c28 100644 --- a/pkg/reporting/options.go +++ b/pkg/reporting/options.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk" "github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters" @@ -44,6 +45,8 @@ type Options struct { JSONExporter *jsonexporter.Options `yaml:"json"` // JSONLExporter contains configuration options for JSONL Exporter Module JSONLExporter *jsonl.Options `yaml:"jsonl"` + // MongoDBExporter containers the configuration options for the MongoDB Exporter Module + MongoDBExporter *mongo.Options `yaml:"mongodb"` HttpClient *retryablehttp.Client `yaml:"-"` OmitRaw bool `yaml:"-"` diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index c6a7d63e1..ddc542863 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -2,6 +2,7 @@ package reporting import ( "fmt" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/mongo" "os" "strings" "sync/atomic" @@ -166,6 +167,13 @@ func New(options *Options, db string, doNotDedupe bool) (Client, error) { } client.exporters = append(client.exporters, exporter) } + if options.MongoDBExporter != nil { + exporter, err := mongo.New(options.MongoDBExporter) + if err != nil { + return nil, errorutil.NewWithErr(err).Wrap(ErrExportClientCreation) + } + client.exporters = append(client.exporters, exporter) + } if doNotDedupe { return client, nil @@ -212,6 +220,7 @@ func CreateConfigIfNotExists() error { SplunkExporter: &splunk.Options{}, JSONExporter: &json_exporter.Options{}, JSONLExporter: &jsonl.Options{}, + MongoDBExporter: &mongo.Options{}, } reportingFile, err := os.Create(reportingConfig) if err != nil {