External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 13:15:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added changes as per code review

Browse Source
This commit is contained in:
Ice3man 2022-12-16 23:10:43 +05:30
parent 7179beab1c
commit 3409f9fca3
10 changed files with 156 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
v2/cmd/nuclei/main.go
View File

@ -295,8 +295,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("cloud", "Cloud",
flagSet.BoolVar(&options.Cloud, "cloud", false, "run scan on nuclei cloud"),
flagSet.StringVarEnv(&options.CloudURL, "cloud-server", "cs", "https://cloud-dev.nuclei.sh", "NUCLEI_CLOUD_SERVER", "nuclei cloud server to use (NUCLEI_CLOUD_SERVER)"),
flagSet.StringVarEnv(&options.CloudAPIKey, "cloud-api-key", "ak", "", "NUCLEI_CLOUD_APIKEY", "api-key for the nuclei cloud server (NUCLEI_CLOUD_APIKEY)"),
flagSet.BoolVarP(&options.ScanList, "list-scan", "ls", false, "list previous cloud scans"),
flagSet.BoolVarP(&options.NoStore, "no-store", "ns", false, "disable scan/output storage on cloud"),
flagSet.IntVarP(&options.OutputLimit, "limit", "ol", 100, "limit the output at a time"),
@ -305,6 +303,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.ListDatasources, "list-datasources", "ld", false, "list cloud datasources"),
flagSet.BoolVarP(&options.ListTargets, "list-targets", "ltr", false, "list cloud targets"),
flagSet.BoolVarP(&options.ListTemplates, "list-templates", "ltm", false, "list cloud templates"),
flagSet.StringVarP(&options.AddDatasource, "add-datasource", "ads", "", "add specified data source (s3,github)"),
flagSet.StringVarP(&options.RemoveDatasource, "remove-datasource", "rds", "", "remove specified data source"),
flagSet.StringVarP(&options.AddTarget, "add-target", "atr", "", "add target(s) to cloud"),
flagSet.StringVarP(&options.AddTemplate, "add-template", "atm", "", "add template(s) to cloud"),

30
v2/internal/runner/cloud.go
View File

@ -19,6 +19,7 @@ import (
func (r *Runner) getScanList(limit int) error {
lastTime := "2099-01-02 15:04:05 +0000 UTC"
var count int
var e error
for {
items, err := r.cloudClient.GetScans(limit, lastTime)
@ -30,15 +31,19 @@ func (r *Runner) getScanList(limit int) error {
break
}
for _, v := range items {
count++
lastTime = v.CreatedAt.String()
res := nucleicloud.PrepareScanListOutput(v)
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(res)
} else {
gologger.Silent().Msgf("%s [%d] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.Timestamp, res.ScanID, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime)
gologger.Silent().Msgf("%d. [%s] [STATUS: %s] [MATCHED: %d] [TARGETS: %d] [TEMPLATES: %d] [DURATION: %s]\n", res.ScanID, res.Timestamp, strings.ToUpper(res.ScanStatus), res.ScanResult, res.Target, res.Template, res.ScanTime)
}
}
}
if count == 0 {
return errors.New("no scan list found")
}
return e
}
@ -92,11 +97,14 @@ func (r *Runner) listDatasources() error {
if err != nil {
return err
}
if len(datasources) == 0 {
return errors.New("no cloud datasource list found")
}
for _, source := range datasources {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else {
gologger.Silent().Msgf("[%s] [%d] [%s] [%s] %s", source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.ID, source.Type, source.Repo, source.Path)
gologger.Silent().Msgf("%d. [%s] [%s] [%s] %s", source.ID, source.Updatedat.Format(nucleicloud.DDMMYYYYhhmmss), source.Type, source.Repo, source.Path)
}
}
return err
@ -107,11 +115,14 @@ func (r *Runner) listTargets() error {
if err != nil {
return err
}
if len(items) == 0 {
return errors.New("no target list found")
}
for _, source := range items {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else {
gologger.Silent().Msgf("[%d] %s (%d)", source.ID, source.Reference, source.Count)
gologger.Silent().Msgf("%d. %s (%d)", source.ID, source.Reference, source.Count)
}
}
return err
@ -122,11 +133,14 @@ func (r *Runner) listTemplates() error {
if err != nil {
return err
}
if len(items) == 0 {
return errors.New("no template list found")
}
for _, source := range items {
if r.options.JSON {
_ = jsoniter.NewEncoder(os.Stdout).Encode(source)
} else {
gologger.Silent().Msgf("[%d] %s", source.ID, source.Reference)
gologger.Silent().Msgf("%d. %s", source.ID, source.Reference)
}
}
return err
@ -217,18 +231,20 @@ func (r *Runner) removeTemplate(item string) error {
}
// initializeCloudDataSources initializes cloud data sources
func (r *Runner) initializeCloudDataSources() error {
if r.options.AwsBucketName != "" {
func (r *Runner) addCloudDataSource(source string) error {
switch source {
case "s3":
token := strings.Join([]string{r.options.AwsAccessKey, r.options.AwsSecretKey, r.options.AwsRegion}, ":")
if _, err := r.processDataSourceItem(r.options.AwsBucketName, token, "s3"); err != nil {
return err
}
}
case "github":
for _, repo := range r.options.GithubTemplateRepo {
if _, err := r.processDataSourceItem(repo, r.options.GithubToken, "github"); err != nil {
return err
}
}
}
return nil
}

7
v2/internal/runner/enumerate.go
View File

@ -33,9 +33,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecuterOptions,
// runCloudEnumeration runs cloud based enumeration
func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) {
count := &atomic.Int64{}
now := time.Now()
defer func() {
gologger.Info().Msgf("Scan execution took %s", time.Since(now))
gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load())
}()
results := &atomic.Bool{}
@ -76,10 +77,14 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudT
return results, err
}
gologger.Info().Msgf("Created task with ID: %d", taskID)
if nostore {
gologger.Info().Msgf("Cloud scan storage: disabled")
}
time.Sleep(3 * time.Second)
err = r.cloudClient.GetResults(taskID, func(re *output.ResultEvent) {
results.CompareAndSwap(false, true)
_ = count.Inc()
if outputErr := r.output.Write(re); outputErr != nil {
gologger.Warning().Msgf("Could not write output: %s", err)

38
v2/internal/runner/nucleicloud/cloud.go
View File

@ -476,6 +476,44 @@ func (c *Client) GetTemplate(ID int64) (io.ReadCloser, error) {
return resp.Body, nil
}
func (c *Client) ExistsTarget(id int64) (ExistsInputResponse, error) {
var item ExistsInputResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/targets/%d/exists", c.baseURL, id), nil)
if err != nil {
return item, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return item, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return item, errors.Wrap(err, "could not decode results")
}
return item, nil
}
func (c *Client) ExistsTemplate(id int64) (ExistsInputResponse, error) {
var item ExistsInputResponse
httpReq, err := retryablehttp.NewRequest(http.MethodGet, fmt.Sprintf("%s/templates/%d/exists", c.baseURL, id), nil)
if err != nil {
return item, errors.Wrap(err, "could not make request")
}
resp, err := c.sendRequest(httpReq)
if err != nil {
return item, errors.Wrap(err, "could not do request")
}
defer resp.Body.Close()
if err := jsoniter.NewDecoder(resp.Body).Decode(&item); err != nil {
return item, errors.Wrap(err, "could not decode results")
}
return item, nil
}
const apiKeyParameter = "X-API-Key"
type errorResponse struct {

4
v2/internal/runner/nucleicloud/types.go
View File

@ -125,3 +125,7 @@ type ListScanOutput struct {
Target int `json:"target"`
Template int `json:"template"`
}
type ExistsInputResponse struct {
Reference string `json:"reference"`
}

59
v2/internal/runner/options.go
View File

@ -160,16 +160,7 @@ func validateOptions(options *types.Options) error {
}
// Verify aws secrets are passed if s3 template bucket passed
if options.AwsBucketName != "" && options.UpdateTemplates {
var missing []string
if options.AwsAccessKey == "" {
missing = append(missing, "AWS_ACCESS_KEY")
}
if options.AwsSecretKey == "" {
missing = append(missing, "AWS_SECRET_KEY")
}
if options.AwsRegion == "" {
missing = append(missing, "AWS_REGION")
}
missing := validateMissingS3Options(options)
if missing != nil {
return fmt.Errorf("aws s3 bucket details are missing. Please provide %s", strings.Join(missing, ","))
}
@ -208,9 +199,52 @@ func validateCloudOptions(options *types.Options) error {
return errors.New("cloud flags cannot be used without cloud option")
}
}
if options.Cloud {
if options.CloudAPIKey == "" {
return errors.New("missing NUCLEI_CLOUD_API env variable")
}
var missing []string
switch options.AddDatasource {
case "s3":
missing = validateMissingS3Options(options)
case "github":
missing = validateMissingGithubOptions(options)
}
if len(missing) > 0 {
return fmt.Errorf("missing %v env variables", strings.Join(missing, ", "))
}
}
return nil
}
func validateMissingS3Options(options *types.Options) []string {
var missing []string
if options.AwsBucketName == "" {
missing = append(missing, "AWS_TEMPLATE_BUCKET")
}
if options.AwsAccessKey == "" {
missing = append(missing, "AWS_ACCESS_KEY")
}
if options.AwsSecretKey == "" {
missing = append(missing, "AWS_SECRET_KEY")
}
if options.AwsRegion == "" {
missing = append(missing, "AWS_REGION")
}
return missing
}
func validateMissingGithubOptions(options *types.Options) []string {
var missing []string
if options.GithubToken == "" {
missing = append(missing, "GITHUB_TOKEN")
}
if len(options.GithubTemplateRepo) == 0 {
missing = append(missing, "GITHUB_TEMPLATE_REPO")
}
return missing
}
// configureOutput configures the output logging levels to be displayed on the screen
func configureOutput(options *types.Options) {
// If the user desires verbose output, show verbose output
@ -291,6 +325,11 @@ func readEnvInputVars(options *types.Options) {
if strings.EqualFold(os.Getenv("NUCLEI_CLOUD"), "true") {
options.Cloud = true
}
if options.CloudURL = os.Getenv("NUCLEI_CLOUD_SERVER"); options.CloudURL == "" {
options.CloudURL = "https://cloud-dev.nuclei.sh"
}
options.CloudAPIKey = os.Getenv("NUCLEI_CLOUD_APIKEY")
options.GithubToken = os.Getenv("GITHUB_TOKEN")
repolist := os.Getenv("GITHUB_TEMPLATE_REPO")
if repolist != "" {

24
v2/internal/runner/runner.go
View File

@ -11,6 +11,7 @@ import (
_ "net/http/pprof"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -183,11 +184,19 @@ func New(options *types.Options) (*Runner, error) {
hmapInput, err := hybrid.New(&hybrid.Options{
Options: options,
NotFoundCallback: func(target string) bool {
parsed, parseErr := strconv.ParseInt(target, 10, 64)
if parseErr != nil {
if err := runner.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Contents: target, Type: "targets"}); err == nil {
runner.cloudTargets = append(runner.cloudTargets, target)
return true
}
return false
}
if exists, err := runner.cloudClient.ExistsTarget(parsed); err == nil {
runner.cloudTargets = append(runner.cloudTargets, exists.Reference)
return true
}
return false
},
})
if err != nil {
@ -413,20 +422,23 @@ func (r *Runner) RunEnumeration() error {
}
var cloudTemplates []string
// Initialize cloud data stores if specified
if r.options.Cloud {
if err := r.initializeCloudDataSources(); err != nil {
return errors.Wrap(err, "could not init cloud data sources")
}
// hook template loading
store.NotFoundCallback = func(template string) bool {
parsed, parseErr := strconv.ParseInt(template, 10, 64)
if parseErr != nil {
if err := r.cloudClient.ExistsDataSourceItem(nucleicloud.ExistsDataSourceItemRequest{Type: "templates", Contents: template}); err == nil {
cloudTemplates = append(cloudTemplates, template)
return true
}
return false
}
if exists, err := r.cloudClient.ExistsTemplate(parsed); err == nil {
cloudTemplates = append(cloudTemplates, exists.Reference)
return true
}
return false
}
}
if r.options.Validate {
if err := store.ValidateTemplates(); err != nil {
@ -481,6 +493,8 @@ func (r *Runner) RunEnumeration() error {
err = r.listTargets()
} else if r.options.ListTemplates {
err = r.listTemplates()
} else if r.options.AddDatasource != "" {
err = r.addCloudDataSource(r.options.AddDatasource)
} else if r.options.RemoveDatasource != "" {
err = r.removeDatasource(r.options.RemoveDatasource)
} else if r.options.AddTarget != "" {

4
v2/pkg/external/customtemplates/github.go vendored
View File

@ -30,7 +30,7 @@ func (customTemplate *customTemplateGithubRepo) Download(location string, ctx co
if !fileutil.FolderExists(clonePath) {
err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
} else {
gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
}
@ -49,7 +49,7 @@ func (customTemplate *customTemplateGithubRepo) Update(location string, ctx cont
}
err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
} else {
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
}

7
v2/pkg/external/customtemplates/templates_provider.go vendored
View File

@ -21,18 +21,21 @@ type Provider interface {
// parseCustomTemplates function reads the options.GithubTemplateRepo list,
// Checks the given repos are valid or not and stores them into runner.CustomTemplates
func ParseCustomTemplates(options *types.Options) []Provider {
if options.Cloud {
return nil
}
var customTemplates []Provider
gitHubClient := getGHClientIncognito()
for _, repoName := range options.GithubTemplateRepo {
owner, repo, err := getOwnerAndRepo(repoName)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
continue
}
githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken)
if err != nil {
gologger.Info().Msgf("%s", err)
gologger.Error().Msgf("%s", err)
continue
}
customTemplateRepo := &customTemplateGithubRepo{

2
v2/pkg/types/types.go
View File

@ -109,6 +109,8 @@ type Options struct {
NoStore bool
// Delete scan
DeleteScan string
// AddDatasource adds a datasource to cloud storage
AddDatasource string
// RemoveDatasource deletes a datasource from cloud storage
RemoveDatasource string
// AddTemplate adds a list of templates to custom datasource