Added changes as per code review

v2/cmd/nuclei/main.go

@@ -295,8 +295,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("cloud", "Cloud",
 		flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
-		flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"),
-		flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"),
 		flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"),
 		flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
 		flagSet.IntVarP(&options.OutputLimit, "limit", "ol", 100, "limit the output at a time"),
@@ -305,6 +303,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.ListDatasources, "list-datasources", "ld", false, "list cloud datasources"),
 		flagSet.BoolVarP(&options.ListTargets, "list-targets", "ltr", false, "list cloud targets"),
 		flagSet.BoolVarP(&options.ListTemplates, "list-templates", "ltm", false, "list cloud templates"),
+		flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"),
 		flagSet.StringVarP(&options.RemoveDatasource, "remove-datasource", "rds", "", "remove specified data source"),
 		flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"),
 		flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"),

v2/internal/runner/cloud.go

@@ -19,6 +19,7 @@ import (
 func (r *Runner) getScanList(limit int) error {
 	lastTime := "2099-01-02 15:04:05 +0000 UTC"
 
+	var count int
 	var e error
 	for {
 		items, err := r.cloudClient.GetScans(limit, lastTime)
@@ -30,15 +31,19 @@ func (r *Runner) getScanList(limit int) error {
 			break
 		}
 		for _, v := range items {
+			count++
 			lastTime = v.CreatedAt.String()
 			res := nucleicloud.PrepareScanListOutput(v)
 			if r.options.JSON {
 				_ = jsoniter.NewEncoder(os.Stdout).Encode(res)
 			} else {
-				gologger.Silent().Msgf("%s [%d] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.Timestamp, res.ScanID, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime)
+				gologger.Silent().Msgf("%d. [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.ScanID, res.Timestamp, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime)
 			}
 		}
 	}
+	if count == 0 {
+		return errors.New("no scan list found")
+	}
 	return e
 }
 
@@ -92,11 +97,14 @@ func (r *Runner) listDatasources() error {
 	if err != nil {
 		return err
 	}
+	if len(datasources) == 0 {
+		return errors.New("no cloud datasource list found")
+	}
 	for _, source := range datasources {
 		if r.options.JSON {
 			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
 		} else {
-			gologger.Silent().Msgf("[%s] [%d] [%s] [%s] %s", source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.ID, source.Type, source.Repo, source.Path)
+			gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path)
 		}
 	}
 	return err
@@ -107,11 +115,14 @@ func (r *Runner) listTargets() error {
 	if err != nil {
 		return err
 	}
+	if len(items) == 0 {
+		return errors.New("no target list found")
+	}
 	for _, source := range items {
 		if r.options.JSON {
 			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
 		} else {
-			gologger.Silent().Msgf("[%d] %s (%d)", source.ID, source.Reference, source.Count)
+			gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count)
 		}
 	}
 	return err
@@ -122,11 +133,14 @@ func (r *Runner) listTemplates() error {
 	if err != nil {
 		return err
 	}
+	if len(items) == 0 {
+		return errors.New("no template list found")
+	}
 	for _, source := range items {
 		if r.options.JSON {
 			_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
 		} else {
-			gologger.Silent().Msgf("[%d] %s", source.ID, source.Reference)
+			gologger.Silent().Msgf("%d. %s", source.ID, source.Reference)
 		}
 	}
 	return err
@@ -217,18 +231,20 @@ func (r *Runner) removeTemplate(item string) error {
 }
 
 // initializeCloudDataSources initializes cloud data sources
-func (r *Runner) initializeCloudDataSources() error {
+func (r *Runner) addCloudDataSource(source string) error {
-	if r.options.AwsBucketName != "" {
+	switch source {
+	case "s3":
 		token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":")
 		if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil {
 			return err
 		}
-	}
+	case "github":
 		for _, repo := range r.options.GithubTemplateRepo {
 			if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil {
 				return err
 			}
 		}
+	}
 	return nil
 }
 

v2/internal/runner/enumerate.go

@@ -33,9 +33,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
 
 // runCloudEnumeration runs cloud based enumeration
 func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) {
+	count := &atomic.Int64{}
 	now := time.Now()
 	defer func() {
-		gologger.Info().Msgf("Scan execution took %s", time.Since(now))
+		gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load())
 	}()
 	results := &atomic.Bool{}
 
@@ -76,10 +77,14 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudT
 		return results, err
 	}
 	gologger.Info().Msgf("Created task with ID: %d", taskID)
+	if nostore {
+		gologger.Info().Msgf("Cloud scan storage: disabled")
+	}
 	time.Sleep(3 * time.Second)
 
 	err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
 		results.CompareAndSwap(false, true)
+		_ = count.Inc()
 
 		if outputErr := r.output.Write(re); outputErr != nil {
 			gologger.Warning().Msgf("Could not write output: %s", err)

v2/internal/runner/nucleicloud/cloud.go

@@ -476,6 +476,44 @@ func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) {
 	return resp.Body, nil
 }
 
+func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) {
+	var item ExistsInputResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil)
+	if err != nil {
+		return item, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return item, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return item, errors.Wrap(err, "could not decode results")
+	}
+	return item, nil
+}
+
+func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) {
+	var item ExistsInputResponse
+	httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil)
+	if err != nil {
+		return item, errors.Wrap(err, "could not make request")
+	}
+
+	resp, err := c.sendRequest(httpReq)
+	if err != nil {
+		return item, errors.Wrap(err, "could not do request")
+	}
+	defer resp.Body.Close()
+
+	if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
+		return item, errors.Wrap(err, "could not decode results")
+	}
+	return item, nil
+}
+
 const apiKeyParameter = "X-API-Key"
 
 type errorResponse struct {

v2/internal/runner/nucleicloud/types.go

@@ -125,3 +125,7 @@ type ListScanOutput struct {
 	Target     int    `json:"target"`
 	Template   int    `json:"template"`
 }
+
+type ExistsInputResponse struct {
+	Reference string `json:"reference"`
+}

v2/internal/runner/options.go

@@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error {
 	}
 	// Verify aws secrets are passed if s3 template bucket passed
 	if options.AwsBucketName != "" && options.UpdateTemplates {
-		var missing []string
+		missing := validateMissingS3Options(options)
-		if options.AwsAccessKey == "" {
-			missing = append(missing, "AWS_ACCESS_KEY")
-		}
-		if options.AwsSecretKey == "" {
-			missing = append(missing, "AWS_SECRET_KEY")
-		}
-		if options.AwsRegion == "" {
-			missing = append(missing, "AWS_REGION")
-		}
 		if missing != nil {
 			return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ","))
 		}
@@ -208,9 +199,52 @@ func validateCloudOptions(options *types.Options) error {
 			return errors.New("cloud flags cannot be used without cloud option")
 		}
 	}
+	if options.Cloud {
+		if options.CloudAPIKey == "" {
+			return errors.New("missing NUCLEI_CLOUD_API env variable")
+		}
+		var missing []string
+		switch options.AddDatasource {
+		case "s3":
+			missing = validateMissingS3Options(options)
+		case "github":
+			missing = validateMissingGithubOptions(options)
+		}
+		if len(missing) > 0 {
+			return fmt.Errorf("missing %v env variables", strings.Join(missing, ", "))
+		}
+	}
 	return nil
 }
 
+func validateMissingS3Options(options *types.Options) []string {
+	var missing []string
+	if options.AwsBucketName == "" {
+		missing = append(missing, "AWS_TEMPLATE_BUCKET")
+	}
+	if options.AwsAccessKey == "" {
+		missing = append(missing, "AWS_ACCESS_KEY")
+	}
+	if options.AwsSecretKey == "" {
+		missing = append(missing, "AWS_SECRET_KEY")
+	}
+	if options.AwsRegion == "" {
+		missing = append(missing, "AWS_REGION")
+	}
+	return missing
+}
+
+func validateMissingGithubOptions(options *types.Options) []string {
+	var missing []string
+	if options.GithubToken == "" {
+		missing = append(missing, "GITHUB_TOKEN")
+	}
+	if len(options.GithubTemplateRepo) == 0 {
+		missing = append(missing, "GITHUB_TEMPLATE_REPO")
+	}
+	return missing
+}
+
 // configureOutput configures the output logging levels to be displayed on the screen
 func configureOutput(options *types.Options) {
 	// If the user desires verbose output, show verbose output
@@ -291,6 +325,11 @@ func readEnvInputVars(options *types.Options) {
 	if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") {
 		options.Cloud = true
 	}
+	if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" {
+		options.CloudURL = "https://cloud-dev.nuclei.sh"
+	}
+	options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_APIKEY")
+
 	options.GithubToken = os.Getenv("GITHUB_TOKEN")
 	repolist := os.Getenv("GITHUB_TEMPLATE_REPO")
 	if repolist != "" {

v2/internal/runner/runner.go

@@ -11,6 +11,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -183,11 +184,19 @@ func New(options *types.Options) (*Runner, error) {
 	hmapInput, err := hybrid.New(&hybrid.Options{
 		Options: options,
 		NotFoundCallback: func(target string) bool {
+			parsed, parseErr := strconv.ParseInt(target, 10, 64)
+			if parseErr != nil {
 				if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil {
 					runner.cloudTargets = append(runner.cloudTargets, target)
 					return true
 				}
 				return false
+			}
+			if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil {
+				runner.cloudTargets = append(runner.cloudTargets, exists.Reference)
+				return true
+			}
+			return false
 		},
 	})
 	if err != nil {
@@ -413,20 +422,23 @@ func (r *Runner) RunEnumeration() error {
 	}
 
 	var cloudTemplates []string
-	// Initialize cloud data stores if specified
 	if r.options.Cloud {
-		if err := r.initializeCloudDataSources(); err != nil {
-			return errors.Wrap(err, "could not init cloud data sources")
-		}
-
 		// hook template loading
 		store.NotFoundCallback = func(template string) bool {
+			parsed, parseErr := strconv.ParseInt(template, 10, 64)
+			if parseErr != nil {
 				if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil {
 					cloudTemplates = append(cloudTemplates, template)
 					return true
 				}
 				return false
 			}
+			if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil {
+				cloudTemplates = append(cloudTemplates, exists.Reference)
+				return true
+			}
+			return false
+		}
 	}
 	if r.options.Validate {
 		if err := store.ValidateTemplates(); err != nil {
@@ -481,6 +493,8 @@ func (r *Runner) RunEnumeration() error {
 			err = r.listTargets()
 		} else if r.options.ListTemplates {
 			err = r.listTemplates()
+		} else if r.options.AddDatasource != "" {
+			err = r.addCloudDataSource(r.options.AddDatasource)
 		} else if r.options.RemoveDatasource != "" {
 			err = r.removeDatasource(r.options.RemoveDatasource)
 		} else if r.options.AddTarget != "" {

v2/pkg/external/customtemplates/github.go

@@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co
 	if !fileutil.FolderExists(clonePath) {
 		err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 		} else {
 			gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
 		}
@@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont
 	}
 	err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
 	if err != nil {
-		gologger.Info().Msgf("%s", err)
+		gologger.Error().Msgf("%s", err)
 	} else {
 		gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
 	}

v2/pkg/external/customtemplates/templates_provider.go

@@ -21,18 +21,21 @@ type Provider interface {
 // parseCustomTemplates function reads the options.GithubTemplateRepo list,
 // Checks the given repos are valid or not and stores them into runner.CustomTemplates
 func ParseCustomTemplates(options *types.Options) []Provider {
+	if options.Cloud {
+		return nil
+	}
 	var customTemplates []Provider
 	gitHubClient := getGHClientIncognito()
 
 	for _, repoName := range options.GithubTemplateRepo {
 		owner, repo, err := getOwnerAndRepo(repoName)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 			continue
 		}
 		githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken)
 		if err != nil {
-			gologger.Info().Msgf("%s", err)
+			gologger.Error().Msgf("%s", err)
 			continue
 		}
 		customTemplateRepo := &customTemplateGithubRepo{

v2/pkg/types/types.go

@@ -109,6 +109,8 @@ type Options struct {
 	NoStore bool
 	// Delete scan
 	DeleteScan string
+	// AddDatasource adds a datasource to cloud storage
+	AddDatasource string
 	// RemoveDatasource deletes a datasource from cloud storage
 	RemoveDatasource string
 	// AddTemplate adds a list of templates to custom datasource