nuclei/v2/internal/runner/runner.go

713 lines
23 KiB
Go
Raw Normal View History

package runner
import (
"bufio"
"bytes"
"context"
2021-11-29 14:38:45 +01:00
"encoding/json"
"fmt"
"io"
"net/http"
_ "net/http/pprof"
"os"
"path/filepath"
"strings"
2021-04-16 16:56:41 +05:30
"time"
"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
"github.com/blang/semver"
2020-08-24 00:16:18 +05:30
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
2022-10-12 22:04:37 -05:00
"github.com/projectdiscovery/ratelimit"
"go.uber.org/atomic"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
2021-02-26 13:13:11 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
2021-10-27 16:50:36 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/core"
2021-10-28 17:45:38 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid"
"github.com/projectdiscovery/nuclei/v2/pkg/input"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
2020-10-18 03:09:24 +02:00
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
2021-04-16 16:56:41 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
2021-09-19 16:26:47 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown"
2021-06-05 18:01:08 +05:30
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
stringsutil "github.com/projectdiscovery/utils/strings"
)
// Runner is a client for running the enumeration process.
type Runner struct {
output output.Writer
interactsh *interactsh.Client
templatesConfig *config.Config
options *types.Options
projectFile *projectfile.ProjectFile
catalog catalog.Catalog
progress progress.Progress
colorizer aurora.Aurora
issuesClient *reporting.Client
hmapInputProvider *hybrid.Input
browser *engine.Browser
ratelimiter *ratelimit.Limiter
hostErrors hosterrorscache.CacheInterface
2021-11-29 14:38:45 +01:00
resumeCfg *types.ResumeCfg
pprofServer *http.Server
Issue 2613 custom template GitHub (#2630) * Add custom template download/update support from github - Accept the -gtr flag to accept the list of custom template repos(public/private) - Accept the -gt flag for github token. It internally sets os.Env variable - Update the flags from - -update to -nuclei-update for nuclei self update - -ut to -tup for template-update - -ud to -tud for custom template location - Add github.go file which has code related to download and update custom templates repos. * Reslove golint and test case error * Take default template from community directory - No need to give explicit community directory path. - Update the integration test to support the change in path * Update functional test script update template flag * Update the path from community to nuclei-template - Revert the code changes that were made to add community directory * remove the comment * Update the interactsh server url for testing * Update race condition command * update race condition cmd to download the templates * Debug integration test failure * update integration test to update templates * Refactor downloadCustomTemplate function. - Remove the log prining instead send the message. * Add test case for custom template repo download * move the download repo for loop into diff function * refactor updateTemplate function. * Create struct for github repos. - Create customtemplate struct for repo. - Add functions to customtemplate * update readme.md file * Refactor the downloadCustomTemplate function - create const variables for github & community as template type - Update gologger to INF - Validate templateUpdate to accept only github & community value. - Validate tempalteUpdate require githubTemplateRepo * Resolve requested changes * go mod update * misc option update * test update * Revert back update-template flag to boolean. - to update community templates `nuclei -ut` - to update custom templates `nuclei -ut -gtr ehsandeep/mobile-nuclei-templates` * Update readme to update flag documentation * Update go.mod Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
2022-11-03 20:27:18 +05:30
customTemplates []customTemplateRepo
cloudClient *nucleicloud.Client
}
const pprofServerAddress = "127.0.0.1:8086"
// New creates a new client for running enumeration process.
func New(options *types.Options) (*Runner, error) {
runner := &Runner{
options: options,
}
if options.HealthCheck {
gologger.Print().Msgf("%s\n", DoHealthCheck(options))
os.Exit(0)
}
if options.Cloud {
runner.cloudClient = nucleicloud.New(options.CloudURL, options.CloudAPIKey)
}
if options.UpdateNuclei {
2021-07-25 03:13:46 +05:30
if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil {
return nil, err
}
return nil, nil
}
if options.Validate {
parsers.ShouldValidate = true
2021-12-01 10:35:18 -06:00
// Does not update the templates when validate flag is used
options.NoUpdateTemplates = true
}
parsers.NoStrictSyntax = options.NoStrictSyntax
Issue 2613 custom template GitHub (#2630) * Add custom template download/update support from github - Accept the -gtr flag to accept the list of custom template repos(public/private) - Accept the -gt flag for github token. It internally sets os.Env variable - Update the flags from - -update to -nuclei-update for nuclei self update - -ut to -tup for template-update - -ud to -tud for custom template location - Add github.go file which has code related to download and update custom templates repos. * Reslove golint and test case error * Take default template from community directory - No need to give explicit community directory path. - Update the integration test to support the change in path * Update functional test script update template flag * Update the path from community to nuclei-template - Revert the code changes that were made to add community directory * remove the comment * Update the interactsh server url for testing * Update race condition command * update race condition cmd to download the templates * Debug integration test failure * update integration test to update templates * Refactor downloadCustomTemplate function. - Remove the log prining instead send the message. * Add test case for custom template repo download * move the download repo for loop into diff function * refactor updateTemplate function. * Create struct for github repos. - Create customtemplate struct for repo. - Add functions to customtemplate * update readme.md file * Refactor the downloadCustomTemplate function - create const variables for github & community as template type - Update gologger to INF - Validate templateUpdate to accept only github & community value. - Validate tempalteUpdate require githubTemplateRepo * Resolve requested changes * go mod update * misc option update * test update * Revert back update-template flag to boolean. - to update community templates `nuclei -ut` - to update custom templates `nuclei -ut -gtr ehsandeep/mobile-nuclei-templates` * Update readme to update flag documentation * Update go.mod Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
2022-11-03 20:27:18 +05:30
// parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos
runner.parseCustomTemplates()
2020-06-25 03:53:37 +05:30
if err := runner.updateTemplates(); err != nil {
gologger.Error().Msgf("Could not update templates: %s\n", err)
2020-06-25 03:53:37 +05:30
}
2021-07-25 03:13:46 +05:30
if options.Headless {
if engine.MustDisableSandbox() {
gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n")
}
2021-07-25 03:13:46 +05:30
browser, err := engine.New(options)
if err != nil {
return nil, err
}
runner.browser = browser
}
2021-06-11 14:44:37 +05:30
runner.catalog = disk.NewCatalog(runner.options.TemplatesDirectory)
var httpclient *retryablehttp.Client
if options.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
var err error
httpclient, err = httpclientpool.Get(options, &httpclientpool.Configuration{})
if err != nil {
return nil, err
}
}
if err := reporting.CreateConfigIfNotExists(); err != nil {
return nil, err
}
reportingOptions, err := createReportingOptions(options)
if err != nil {
return nil, err
2021-06-05 18:01:08 +05:30
}
if reportingOptions != nil && httpclient != nil {
reportingOptions.HttpClient = httpclient
}
if reportingOptions != nil {
2021-07-07 19:23:25 +05:30
client, err := reporting.New(reportingOptions, options.ReportingDB)
if err != nil {
return nil, errors.Wrap(err, "could not create issue reporting client")
}
2021-07-07 19:23:25 +05:30
runner.issuesClient = client
}
// output coloring
useColor := !options.NoColor
runner.colorizer = aurora.NewAurora(useColor)
templates.Colorizer = runner.colorizer
templates.SeverityColorizer = colorizer.New(runner.colorizer)
if options.EnablePprof {
server := &http.Server{
Addr: pprofServerAddress,
Handler: http.DefaultServeMux,
}
gologger.Info().Msgf("Listening pprof debug server on: %s", pprofServerAddress)
runner.pprofServer = server
go func() {
2022-03-07 10:28:25 +05:30
_ = server.ListenAndServe()
}()
}
if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
2020-06-25 03:53:37 +05:30
os.Exit(0)
}
// Initialize the input source
hmapInput, err := hybrid.New(options)
if err != nil {
return nil, errors.Wrap(err, "could not create input provider")
2020-07-23 20:19:19 +02:00
}
runner.hmapInputProvider = hmapInput
2020-07-23 20:19:19 +02:00
2020-04-04 18:21:05 +05:30
// Create the output file if asked
outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.StoreResponse, options.Output, options.TraceLogFile, options.ErrorLogFile, options.StoreResponseDir)
2020-12-29 18:15:27 +05:30
if err != nil {
return nil, errors.Wrap(err, "could not create output file")
2020-04-04 18:21:05 +05:30
}
2021-02-26 13:13:11 +05:30
runner.output = outputWriter
2020-07-23 20:19:19 +02:00
if options.JSON && options.EnableProgressBar {
options.StatsJSON = true
}
if options.StatsJSON {
options.EnableProgressBar = true
}
// Creates the progress tracking object
var progressErr error
runner.progress, progressErr = progress.NewStatsTicker(options.StatsInterval, options.EnableProgressBar, options.StatsJSON, options.Metrics, options.MetricsPort)
if progressErr != nil {
return nil, progressErr
}
2020-07-23 20:19:19 +02:00
// create project file if requested or load the existing one
2020-10-17 02:10:47 +02:00
if options.Project {
2020-10-30 13:06:05 +01:00
var projectFileErr error
runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: utils.IsBlank(options.ProjectPath)})
2020-10-30 13:06:05 +01:00
if projectFileErr != nil {
return nil, projectFileErr
2020-10-15 23:39:00 +02:00
}
}
2021-11-29 14:38:45 +01:00
// create the resume configuration structure
resumeCfg := types.NewResumeCfg()
if runner.options.ShouldLoadResume() {
gologger.Info().Msg("Resuming from save checkpoint")
file, err := os.ReadFile(runner.options.Resume)
2021-11-29 14:38:45 +01:00
if err != nil {
return nil, err
}
err = json.Unmarshal([]byte(file), &resumeCfg)
if err != nil {
return nil, err
}
resumeCfg.Compile()
2021-11-29 14:38:45 +01:00
}
runner.resumeCfg = resumeCfg
opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress)
opts.Debug = runner.options.Debug
opts.NoColor = runner.options.NoColor
if options.InteractshURL != "" {
opts.ServerURL = options.InteractshURL
}
opts.Authorization = options.InteractshToken
opts.CacheSize = int64(options.InteractionsCacheSize)
opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second
opts.CooldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second
opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second
opts.NoInteractsh = runner.options.NoInteractsh
opts.StopAtFirstMatch = runner.options.StopAtFirstMatch
opts.Debug = runner.options.Debug
opts.DebugRequest = runner.options.DebugRequests
opts.DebugResponse = runner.options.DebugResponse
if httpclient != nil {
opts.HTTPClient = httpclient
}
interactshClient, err := interactsh.New(opts)
if err != nil {
gologger.Error().Msgf("Could not create interactsh client: %s", err)
} else {
runner.interactsh = interactshClient
2021-04-16 16:56:41 +05:30
}
if options.RateLimitMinute > 0 {
runner.ratelimiter = ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute)
} else if options.RateLimit > 0 {
runner.ratelimiter = ratelimit.New(context.Background(), uint(options.RateLimit), time.Second)
} else {
runner.ratelimiter = ratelimit.NewUnlimited(context.Background())
}
return runner, nil
}
func createReportingOptions(options *types.Options) (*reporting.Options, error) {
var reportingOptions *reporting.Options
if options.ReportingConfig != "" {
file, err := os.Open(options.ReportingConfig)
if err != nil {
return nil, errors.Wrap(err, "could not open reporting config file")
}
reportingOptions = &reporting.Options{}
if err := yamlwrapper.DecodeAndValidate(file, reportingOptions); err != nil {
file.Close()
return nil, errors.Wrap(err, "could not parse reporting config file")
}
file.Close()
}
2021-09-19 16:26:47 +05:30
if options.MarkdownExportDirectory != "" {
if reportingOptions != nil {
2021-09-19 16:26:47 +05:30
reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
} else {
reportingOptions = &reporting.Options{}
2021-09-19 16:26:47 +05:30
reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
}
}
if options.SarifExport != "" {
if reportingOptions != nil {
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
}
}
return reportingOptions, nil
}
// Close releases all the resources and cleans up
2020-04-04 18:21:05 +05:30
func (r *Runner) Close() {
2020-09-11 21:35:29 +05:30
if r.output != nil {
r.output.Close()
}
if r.projectFile != nil {
r.projectFile.Close()
2020-10-15 23:39:00 +02:00
}
r.hmapInputProvider.Close()
protocolinit.Close()
if r.pprofServer != nil {
_ = r.pprofServer.Shutdown(context.Background())
}
2020-04-04 18:21:05 +05:30
}
// RunEnumeration sets up the input layer for giving input nuclei.
// binary and runs the actual enumeration
func (r *Runner) RunEnumeration() error {
2021-03-22 15:00:26 +05:30
defer r.Close()
// If user asked for new templates to be executed, collect the list from the templates' directory.
if r.options.NewTemplates {
2021-03-09 15:00:22 +05:30
templatesLoaded, err := r.readNewTemplatesFile()
if err != nil {
return errors.Wrap(err, "could not get newly added templates")
}
2021-03-09 15:00:22 +05:30
r.options.Templates = append(r.options.Templates, templatesLoaded...)
}
if len(r.options.NewTemplatesWithVersion) > 0 {
minVersion, err := semver.Parse("8.8.4")
if err != nil {
return errors.Wrap(err, "could not parse minimum version")
}
latestVersion, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
if err != nil {
return errors.Wrap(err, "could not get latest version")
}
for _, version := range r.options.NewTemplatesWithVersion {
current, err := semver.Parse(strings.Trim(version, "v"))
if err != nil {
return errors.Wrap(err, "could not parse current version")
}
if !(current.GT(minVersion) && current.LTE(latestVersion)) {
return fmt.Errorf("version should be greater than %s and less than %s", minVersion, latestVersion)
}
templatesLoaded, err := r.readNewTemplatesWithVersionFile(fmt.Sprintf("v%s", current))
if err != nil {
return errors.Wrap(err, "could not get newly added templates for "+current.String())
}
r.options.Templates = append(r.options.Templates, templatesLoaded...)
}
}
2021-12-01 10:35:18 -06:00
// Exclude ignored file for validation
if !r.options.Validate {
ignoreFile := config.ReadIgnoreFile()
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
}
var cache *hosterrorscache.Cache
2021-09-01 01:09:16 +05:30
if r.options.MaxHostError > 0 {
cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount)
cache.SetVerbose(r.options.Verbose)
}
r.hostErrors = cache
2021-10-27 16:50:36 +05:30
// Create the executer options which will be used throughout the execution
// stage by the nuclei engine modules.
2021-07-01 14:36:40 +05:30
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
HostErrorsCache: cache,
Colorizer: r.colorizer,
2021-11-29 14:38:45 +01:00
ResumeCfg: r.resumeCfg,
ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers),
InputHelper: input.NewHelper(),
2021-07-01 14:36:40 +05:30
}
2021-10-27 16:50:36 +05:30
engine := core.New(r.options)
engine.SetExecuterOptions(executerOpts)
workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
return errors.Wrap(err, "Could not create loader.")
}
executerOpts.WorkflowLoader = workflowLoader
templateConfig := r.templatesConfig
if templateConfig == nil {
templateConfig = &config.Config{}
}
store, err := loader.New(loader.NewConfig(r.options, templateConfig, r.catalog, executerOpts))
if err != nil {
return errors.Wrap(err, "could not load templates from config")
2020-08-02 15:48:10 +02:00
}
if r.options.Validate {
if err := store.ValidateTemplates(); err != nil {
2021-08-27 17:06:06 +03:00
return err
}
if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 && stats.GetValue(parsers.RuntimeWarningsStats) == 0 {
gologger.Info().Msgf("All templates validated successfully\n")
} else {
return errors.New("encountered errors while performing template validation")
}
2021-07-07 19:23:25 +05:30
return nil // exit
}
store.Load()
// add the hosts from the metadata queries of loaded templates into input provider
if r.options.Uncover && len(r.options.UncoverQuery) == 0 {
ret := uncover.GetUncoverTargetsFromMetadata(store.Templates(), r.options.UncoverDelay, r.options.UncoverLimit, r.options.UncoverField)
for host := range ret {
r.hmapInputProvider.Set(host)
}
}
// list all templates
if r.options.TemplateList {
r.listAvailableStoreTemplates(store)
os.Exit(0)
}
2021-10-28 23:17:05 +05:30
r.displayExecutionInfo(store)
// If not explicitly disabled, check if http based protocols
// are used and if inputs are non-http to pre-perform probing
// of urls and storing them for execution.
if !r.options.DisableHTTPProbe && loader.IsHTTPBasedProtocolUsed(store) && r.isInputNonHTTP() {
inputHelpers, err := r.initializeTemplatesHTTPInput()
if err != nil {
return errors.Wrap(err, "could not probe http input")
}
executerOpts.InputHelper.InputsHTTP = inputHelpers
}
2022-04-18 17:21:33 -05:00
enumeration := false
var results *atomic.Bool
if r.options.Cloud {
if r.options.ScanList {
err = r.getScanList()
} else if r.options.DeleteScan != "" {
err = r.deleteScan(r.options.DeleteScan)
} else if r.options.ScanOutput != "" {
err = r.getResults(r.options.ScanOutput)
} else {
gologger.Info().Msgf("Running scan on cloud with URL %s", r.options.CloudURL)
results, err = r.runCloudEnumeration(store, r.options.NoStore)
enumeration = true
}
} else {
results, err = r.runStandardEnumeration(executerOpts, store, engine)
enumeration = true
}
if !enumeration {
return err
}
if r.interactsh != nil {
matched := r.interactsh.Close()
if matched {
2022-09-19 08:38:52 +02:00
results.CompareAndSwap(false, true)
}
}
r.progress.Stop()
if executerOpts.InputHelper != nil {
_ = executerOpts.InputHelper.Close()
}
if r.issuesClient != nil {
r.issuesClient.Close()
}
if !results.Load() {
gologger.Info().Msgf("No results found. Better luck next time!")
}
if r.browser != nil {
r.browser.Close()
}
return err
}
func (r *Runner) isInputNonHTTP() bool {
var nonURLInput bool
r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
if !strings.Contains(value.Input, "://") {
nonURLInput = true
return false
}
return true
})
return nonURLInput
}
func (r *Runner) executeSmartWorkflowInput(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
r.progress.Init(r.hmapInputProvider.Count(), 0, 0)
service, err := automaticscan.New(automaticscan.Options{
ExecuterOpts: executerOpts,
Store: store,
Engine: engine,
Target: r.hmapInputProvider,
})
if err != nil {
2022-04-19 16:14:49 +05:30
return nil, errors.Wrap(err, "could not create automatic scan service")
}
service.Execute()
result := &atomic.Bool{}
result.Store(service.Close())
return result, nil
}
func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
2021-07-05 17:29:45 +05:30
var unclusteredRequests int64
2021-07-01 14:36:40 +05:30
for _, template := range store.Templates() {
2021-01-15 14:17:34 +05:30
// workflows will dynamically adjust the totals while running, as
// it can't be known in advance which requests will be called
2021-01-15 14:17:34 +05:30
if len(template.Workflows) > 0 {
continue
}
unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count()
2021-01-15 14:17:34 +05:30
}
if r.options.VerboseVerbose {
2021-07-01 14:36:40 +05:30
for _, template := range store.Templates() {
r.logAvailableTemplate(template.Path)
}
for _, template := range store.Workflows() {
r.logAvailableTemplate(template.Path)
}
}
2021-01-15 14:17:34 +05:30
// Cluster the templates first because we want info on how many
// templates did we cluster for showing to user in CLI
originalTemplatesCount := len(store.Templates())
finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions())
finalTemplates = append(finalTemplates, store.Workflows()...)
2021-06-14 17:14:16 +05:30
var totalRequests int64
2021-01-15 14:17:34 +05:30
for _, t := range finalTemplates {
if len(t.Workflows) > 0 {
continue
}
totalRequests += int64(t.Executer.Requests()) * r.hmapInputProvider.Count()
2021-01-15 14:17:34 +05:30
}
if totalRequests < unclusteredRequests {
2021-07-08 00:41:22 +05:30
gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests)
2021-01-15 14:17:34 +05:30
}
2021-07-01 14:36:40 +05:30
workflowCount := len(store.Workflows())
templateCount := originalTemplatesCount + workflowCount
2020-08-02 18:33:55 +02:00
// 0 matches means no templates were found in directory
2020-08-02 18:33:55 +02:00
if templateCount == 0 {
return &atomic.Bool{}, errors.New("no valid templates were found")
}
// tracks global progress and captures stdout/stderr until p.Wait finishes
r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true)
return results, nil
}
2021-10-28 23:17:05 +05:30
// displayExecutionInfo displays misc info about the nuclei engine execution
func (r *Runner) displayExecutionInfo(store *loader.Store) {
// Display stats for any loaded templates' syntax warnings or errors
stats.Display(parsers.SyntaxWarningStats)
stats.Display(parsers.SyntaxErrorStats)
stats.Display(parsers.RuntimeWarningsStats)
2021-10-28 23:17:05 +05:30
builder := &strings.Builder{}
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
builder.WriteString(" (")
if strings.Contains(config.Version, "-dev") {
builder.WriteString(r.colorizer.Blue("development").String())
} else if config.Version == r.templatesConfig.NucleiLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr := builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic
builder.WriteString(" (")
if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr = builder.String()
builder.Reset()
if r.templatesConfig != nil {
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr)
}
if len(store.Templates()) > 0 {
gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates())
gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates()))
}
if len(store.Workflows()) > 0 {
gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows()))
}
gologger.Info().Msgf("Targets loaded for scan: %d", r.hmapInputProvider.Count())
2021-10-28 23:17:05 +05:30
}
func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) {
resp, err := http.DefaultClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New("version not found")
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
templatesList := []string{}
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
text := scanner.Text()
if text == "" {
continue
}
if isTemplate(text) {
templatesList = append(templatesList, text)
}
}
return templatesList, nil
}
2021-10-28 23:17:05 +05:30
// readNewTemplatesFile reads newly added templates from directory if it exists
func (r *Runner) readNewTemplatesFile() ([]string, error) {
2021-08-27 23:23:01 +05:30
if r.templatesConfig == nil {
return nil, nil
}
additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
file, err := os.Open(additionsFile)
if err != nil {
return nil, err
}
defer file.Close()
2021-03-09 15:00:22 +05:30
templatesList := []string{}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
continue
}
if isTemplate(text) {
templatesList = append(templatesList, text)
}
}
2021-03-09 15:00:22 +05:30
return templatesList, nil
}
2021-07-08 15:15:26 +05:30
// countNewTemplates returns the number of newly added templates
2021-07-08 15:15:26 +05:30
func (r *Runner) countNewTemplates() int {
2021-08-27 23:23:01 +05:30
if r.templatesConfig == nil {
return 0
}
additionsFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".new-additions")
2021-07-08 15:15:26 +05:30
file, err := os.Open(additionsFile)
if err != nil {
return 0
}
defer file.Close()
count := 0
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := scanner.Text()
if text == "" {
continue
}
if isTemplate(text) {
count++
}
2021-07-08 15:15:26 +05:30
}
return count
}
2021-11-29 14:38:45 +01:00
func isTemplate(filename string) bool {
return stringsutil.EqualFoldAny(filepath.Ext(filename), templates.TemplateExtension)
}
2021-11-29 14:38:45 +01:00
// SaveResumeConfig to file
func (r *Runner) SaveResumeConfig(path string) error {
2022-07-26 15:00:15 +02:00
resumeCfgClone := r.resumeCfg.Clone()
resumeCfgClone.ResumeFrom = resumeCfgClone.Current
data, _ := json.MarshalIndent(resumeCfgClone, "", "\t")
2021-11-29 14:38:45 +01:00
return os.WriteFile(path, data, os.ModePerm)
2021-11-29 14:38:45 +01:00
}