diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c0e380b16..7997c949d 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -295,8 +295,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("cloud", "Cloud", flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"), - flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"), - flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"), flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"), flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"), flagSet.IntVarP(&options.OutputLimit, "limit", "ol", 100, "limit the output at a time"), @@ -305,6 +303,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ListDatasources, "list-datasources", "ld", false, "list cloud datasources"), flagSet.BoolVarP(&options.ListTargets, "list-targets", "ltr", false, "list cloud targets"), flagSet.BoolVarP(&options.ListTemplates, "list-templates", "ltm", false, "list cloud templates"), + flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"), flagSet.StringVarP(&options.RemoveDatasource, "remove-datasource", "rds", "", "remove specified data source"), flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"), flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"), diff --git a/v2/internal/runner/cloud.go b/v2/internal/runner/cloud.go index 0c92c051c..5aafd9c15 100644 --- a/v2/internal/runner/cloud.go +++ b/v2/internal/runner/cloud.go @@ -19,6 +19,7 @@ import ( func (r *Runner) getScanList(limit int) error { lastTime := "2099-01-02 15:04:05 +0000 UTC" + var count int var e error for { items, err := r.cloudClient.GetScans(limit, lastTime) @@ -30,15 +31,19 @@ func (r *Runner) getScanList(limit int) error { break } for _, v := range items { + count++ lastTime = v.CreatedAt.String() res := nucleicloud.PrepareScanListOutput(v) if r.options.JSON { _ = jsoniter.NewEncoder(os.Stdout).Encode(res) } else { - gologger.Silent().Msgf("%s [%d] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.Timestamp, res.ScanID, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime) + gologger.Silent().Msgf("%d. [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.ScanID, res.Timestamp, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime) } } } + if count == 0 { + return errors.New("no scan list found") + } return e } @@ -92,11 +97,14 @@ func (r *Runner) listDatasources() error { if err != nil { return err } + if len(datasources) == 0 { + return errors.New("no cloud datasource list found") + } for _, source := range datasources { if r.options.JSON { _ = jsoniter.NewEncoder(os.Stdout).Encode(source) } else { - gologger.Silent().Msgf("[%s] [%d] [%s] [%s] %s", source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.ID, source.Type, source.Repo, source.Path) + gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path) } } return err @@ -107,11 +115,14 @@ func (r *Runner) listTargets() error { if err != nil { return err } + if len(items) == 0 { + return errors.New("no target list found") + } for _, source := range items { if r.options.JSON { _ = jsoniter.NewEncoder(os.Stdout).Encode(source) } else { - gologger.Silent().Msgf("[%d] %s (%d)", source.ID, source.Reference, source.Count) + gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count) } } return err @@ -122,11 +133,14 @@ func (r *Runner) listTemplates() error { if err != nil { return err } + if len(items) == 0 { + return errors.New("no template list found") + } for _, source := range items { if r.options.JSON { _ = jsoniter.NewEncoder(os.Stdout).Encode(source) } else { - gologger.Silent().Msgf("[%d] %s", source.ID, source.Reference) + gologger.Silent().Msgf("%d. %s", source.ID, source.Reference) } } return err @@ -217,16 +231,18 @@ func (r *Runner) removeTemplate(item string) error { } // initializeCloudDataSources initializes cloud data sources -func (r *Runner) initializeCloudDataSources() error { - if r.options.AwsBucketName != "" { +func (r *Runner) addCloudDataSource(source string) error { + switch source { + case "s3": token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":") if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil { return err } - } - for _, repo := range r.options.GithubTemplateRepo { - if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil { - return err + case "github": + for _, repo := range r.options.GithubTemplateRepo { + if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil { + return err + } } } return nil diff --git a/v2/internal/runner/enumerate.go b/v2/internal/runner/enumerate.go index 13c17fd1b..34979d468 100644 --- a/v2/internal/runner/enumerate.go +++ b/v2/internal/runner/enumerate.go @@ -33,9 +33,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions, // runCloudEnumeration runs cloud based enumeration func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) { + count := &atomic.Int64{} now := time.Now() defer func() { - gologger.Info().Msgf("Scan execution took %s", time.Since(now)) + gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load()) }() results := &atomic.Bool{} @@ -76,10 +77,14 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudT return results, err } gologger.Info().Msgf("Created task with ID: %d", taskID) + if nostore { + gologger.Info().Msgf("Cloud scan storage: disabled") + } time.Sleep(3 * time.Second) err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) { results.CompareAndSwap(false, true) + _ = count.Inc() if outputErr := r.output.Write(re); outputErr != nil { gologger.Warning().Msgf("Could not write output: %s", err) diff --git a/v2/internal/runner/nucleicloud/cloud.go b/v2/internal/runner/nucleicloud/cloud.go index 34cd0ba2e..bd9389ab6 100644 --- a/v2/internal/runner/nucleicloud/cloud.go +++ b/v2/internal/runner/nucleicloud/cloud.go @@ -476,6 +476,44 @@ func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) { return resp.Body, nil } +func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) { + var item ExistsInputResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil) + if err != nil { + return item, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return item, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return item, errors.Wrap(err, "could not decode results") + } + return item, nil +} + +func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) { + var item ExistsInputResponse + httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil) + if err != nil { + return item, errors.Wrap(err, "could not make request") + } + + resp, err := c.sendRequest(httpReq) + if err != nil { + return item, errors.Wrap(err, "could not do request") + } + defer resp.Body.Close() + + if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil { + return item, errors.Wrap(err, "could not decode results") + } + return item, nil +} + const apiKeyParameter = "X-API-Key" type errorResponse struct { diff --git a/v2/internal/runner/nucleicloud/types.go b/v2/internal/runner/nucleicloud/types.go index 85dd166c2..2dd44c4db 100644 --- a/v2/internal/runner/nucleicloud/types.go +++ b/v2/internal/runner/nucleicloud/types.go @@ -125,3 +125,7 @@ type ListScanOutput struct { Target int `json:"target"` Template int `json:"template"` } + +type ExistsInputResponse struct { + Reference string `json:"reference"` +} diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 84cf9273b..c807825fd 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error { } // Verify aws secrets are passed if s3 template bucket passed if options.AwsBucketName != "" && options.UpdateTemplates { - var missing []string - if options.AwsAccessKey == "" { - missing = append(missing, "AWS_ACCESS_KEY") - } - if options.AwsSecretKey == "" { - missing = append(missing, "AWS_SECRET_KEY") - } - if options.AwsRegion == "" { - missing = append(missing, "AWS_REGION") - } + missing := validateMissingS3Options(options) if missing != nil { return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ",")) } @@ -208,9 +199,52 @@ func validateCloudOptions(options *types.Options) error { return errors.New("cloud flags cannot be used without cloud option") } } + if options.Cloud { + if options.CloudAPIKey == "" { + return errors.New("missing NUCLEI_CLOUD_API env variable") + } + var missing []string + switch options.AddDatasource { + case "s3": + missing = validateMissingS3Options(options) + case "github": + missing = validateMissingGithubOptions(options) + } + if len(missing) > 0 { + return fmt.Errorf("missing %v env variables", strings.Join(missing, ", ")) + } + } return nil } +func validateMissingS3Options(options *types.Options) []string { + var missing []string + if options.AwsBucketName == "" { + missing = append(missing, "AWS_TEMPLATE_BUCKET") + } + if options.AwsAccessKey == "" { + missing = append(missing, "AWS_ACCESS_KEY") + } + if options.AwsSecretKey == "" { + missing = append(missing, "AWS_SECRET_KEY") + } + if options.AwsRegion == "" { + missing = append(missing, "AWS_REGION") + } + return missing +} + +func validateMissingGithubOptions(options *types.Options) []string { + var missing []string + if options.GithubToken == "" { + missing = append(missing, "GITHUB_TOKEN") + } + if len(options.GithubTemplateRepo) == 0 { + missing = append(missing, "GITHUB_TEMPLATE_REPO") + } + return missing +} + // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -291,6 +325,11 @@ func readEnvInputVars(options *types.Options) { if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") { options.Cloud = true } + if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" { + options.CloudURL = "https://cloud-dev.nuclei.sh" + } + options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_APIKEY") + options.GithubToken = os.Getenv("GITHUB_TOKEN") repolist := os.Getenv("GITHUB_TEMPLATE_REPO") if repolist != "" { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3aa2e2710..4a3e51c87 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,6 +11,7 @@ import ( _ "net/http/pprof" "os" "path/filepath" + "strconv" "strings" "time" @@ -183,8 +184,16 @@ func New(options *types.Options) (*Runner, error) { hmapInput, err := hybrid.New(&hybrid.Options{ Options: options, NotFoundCallback: func(target string) bool { - if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil { - runner.cloudTargets = append(runner.cloudTargets, target) + parsed, parseErr := strconv.ParseInt(target, 10, 64) + if parseErr != nil { + if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil { + runner.cloudTargets = append(runner.cloudTargets, target) + return true + } + return false + } + if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil { + runner.cloudTargets = append(runner.cloudTargets, exists.Reference) return true } return false @@ -413,16 +422,19 @@ func (r *Runner) RunEnumeration() error { } var cloudTemplates []string - // Initialize cloud data stores if specified if r.options.Cloud { - if err := r.initializeCloudDataSources(); err != nil { - return errors.Wrap(err, "could not init cloud data sources") - } - // hook template loading store.NotFoundCallback = func(template string) bool { - if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil { - cloudTemplates = append(cloudTemplates, template) + parsed, parseErr := strconv.ParseInt(template, 10, 64) + if parseErr != nil { + if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil { + cloudTemplates = append(cloudTemplates, template) + return true + } + return false + } + if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil { + cloudTemplates = append(cloudTemplates, exists.Reference) return true } return false @@ -481,6 +493,8 @@ func (r *Runner) RunEnumeration() error { err = r.listTargets() } else if r.options.ListTemplates { err = r.listTemplates() + } else if r.options.AddDatasource != "" { + err = r.addCloudDataSource(r.options.AddDatasource) } else if r.options.RemoveDatasource != "" { err = r.removeDatasource(r.options.RemoveDatasource) } else if r.options.AddTarget != "" { diff --git a/v2/pkg/external/customtemplates/github.go b/v2/pkg/external/customtemplates/github.go index 326ab1c8c..550d5fe06 100644 --- a/v2/pkg/external/customtemplates/github.go +++ b/v2/pkg/external/customtemplates/github.go @@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co if !fileutil.FolderExists(clonePath) { err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) } else { gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath) } @@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont } err := customTemplate.pullChanges(clonePath, customTemplate.githubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) } else { gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame) } diff --git a/v2/pkg/external/customtemplates/templates_provider.go b/v2/pkg/external/customtemplates/templates_provider.go index de6a8e972..e4b5f0ed5 100644 --- a/v2/pkg/external/customtemplates/templates_provider.go +++ b/v2/pkg/external/customtemplates/templates_provider.go @@ -21,18 +21,21 @@ type Provider interface { // parseCustomTemplates function reads the options.GithubTemplateRepo list, // Checks the given repos are valid or not and stores them into runner.CustomTemplates func ParseCustomTemplates(options *types.Options) []Provider { + if options.Cloud { + return nil + } var customTemplates []Provider gitHubClient := getGHClientIncognito() for _, repoName := range options.GithubTemplateRepo { owner, repo, err := getOwnerAndRepo(repoName) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) continue } githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken) if err != nil { - gologger.Info().Msgf("%s", err) + gologger.Error().Msgf("%s", err) continue } customTemplateRepo := &customTemplateGithubRepo{ diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 608300dd0..9f79021a2 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -109,6 +109,8 @@ type Options struct { NoStore bool // Delete scan DeleteScan string + // AddDatasource adds a datasource to cloud storage + AddDatasource string // RemoveDatasource deletes a datasource from cloud storage RemoveDatasource string // AddTemplate adds a list of templates to custom datasource