External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 05:05:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Performance improvements + wappalyzer as default + misc

Browse Source
This commit is contained in:
Ice3man 2022-03-08 12:43:24 +05:30
parent d9a121344c
commit ffe4fea237
2 changed files with 100 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
v2/pkg/core/engine.go
View File

@ -57,3 +57,8 @@ func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) {
func (e *Engine) ExecuterOptions() protocols.ExecuterOptions {
return e.executerOpts
}
// WorkPool returns the worker pool for the engine
func (e *Engine) WorkPool() *WorkPool {
return e.workPool
}

187
v2/pkg/protocols/common/smartworkflow/workflow.go
View File

@ -13,19 +13,23 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/retryablehttp-go"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
// Service is a service for automatic smart workflow execution
type Service struct {
opts protocols.ExecuterOptions
store *loader.Store
engine *core.Engine
target core.InputProvider
wappalyzer *wappalyzer.Wappalyze
opts protocols.ExecuterOptions
store *loader.Store
engine *core.Engine
target core.InputProvider
wappalyzer *wappalyzer.Wappalyze
childExecuter *core.ChildExecuter
httpclient *retryablehttp.Client
results bool
results bool
allTemplates []string
}
// Options contains configuration options for smart workflow service
@ -41,10 +45,13 @@ const (
ModeWorkflow = "workflow"
ModeWappalyzer = "wappalyzer"
ModeAll = "all"
ModeDefault = "default"
)
func Modes() string {
builder := &strings.Builder{}
builder.WriteString(ModeDefault)
builder.WriteString(",")
builder.WriteString(ModeWorkflow)
builder.WriteString(",")
builder.WriteString(ModeWappalyzer)
@ -59,17 +66,43 @@ func New(opts Options) (*Service, error) {
if err != nil {
return nil, err
}
// Collect path for default directories we want to look for templates in
var allTemplates []string
for _, directory := range defaultTemplatesDirectories {
templates, err := opts.ExecuterOpts.Catalog.GetTemplatePath(directory)
if err != nil {
return nil, errors.Wrap(err, "could not get templates in directory")
}
allTemplates = append(allTemplates, templates...)
}
childExecuter := opts.Engine.ChildExecuter()
httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
Connection: &httpclientpool.ConnectionConfiguration{DisableKeepAlive: true},
})
if err != nil {
return nil, errors.Wrap(err, "could not get http client")
}
return &Service{
opts: opts.ExecuterOpts,
store: opts.Store,
engine: opts.Engine,
target: opts.Target,
wappalyzer: wappalyzer,
opts: opts.ExecuterOpts,
store: opts.Store,
engine: opts.Engine,
target: opts.Target,
wappalyzer: wappalyzer,
allTemplates: allTemplates,
childExecuter: childExecuter,
httpclient: httpclient,
}, nil
}
// Close closes the service
func (s *Service) Close() bool {
results := s.childExecuter.Close()
if results.Load() {
s.results = true
}
return s.results
}
@ -80,30 +113,21 @@ func (s *Service) Execute(mode string) {
gologger.Error().Msgf("Could not execute workflow based templates: %s", err)
}
}
wappalyzerFunc := func() map[string][]string {
mapping, err := s.executeWappalyzerTechDetection()
if err != nil {
wappalyzerFunc := func() {
if err := s.executeWappalyzerTechDetection(); err != nil {
gologger.Error().Msgf("Could not execute wappalyzer based detection: %s", err)
}
return mapping
}
modeParts := strings.Split(mode, ",")
for _, value := range modeParts {
switch value {
case ModeWorkflow:
workflowFunc()
case ModeWappalyzer:
mapping := wappalyzerFunc()
if err := s.executeDiscoveredHostTags(mapping); err != nil {
gologger.Error().Msgf("Could not execute discovered tags from wappalyzer: %s", err)
}
case ModeWappalyzer, ModeDefault:
wappalyzerFunc()
case ModeAll:
workflowFunc()
wappalyzerMapping := wappalyzerFunc()
if err := s.executeDiscoveredHostTags(wappalyzerMapping); err != nil {
gologger.Error().Msgf("Could not execute discovered tags from technologies: %s", err)
}
wappalyzerFunc()
default:
gologger.Error().Msgf("Invalid mode value provided to smartworkflows: %s", value)
}
@ -139,78 +163,57 @@ const maxDefaultBody = 2 * 1024 * 1024
// technologies detection on inputs which returns tech.
//
// The returned tags are then used for further execution.
func (s *Service) executeWappalyzerTechDetection() (map[string][]string, error) {
httpclient, err := httpclientpool.Get(s.opts.Options, &httpclientpool.Configuration{
Connection: &httpclientpool.ConnectionConfiguration{DisableKeepAlive: true},
})
if err != nil {
return nil, errors.Wrap(err, "could not get http client")
}
func (s *Service) executeWappalyzerTechDetection() error {
gologger.Info().Msgf("[workflow] Executing wappalyzer based tech detection on inputs")
hostTagsMappings := make(map[string][]string)
// Iterate through each target making http request and identifying fingerprints
inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol)
s.target.Scan(func(value string) {
req, err := retryablehttp.NewRequest(http.MethodGet, value, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", uarand.GetRandom())
inputPool.WaitGroup.Add()
resp, err := httpclient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
return
}
reader := io.LimitReader(resp.Body, maxDefaultBody)
data, err := ioutil.ReadAll(reader)
if err != nil {
resp.Body.Close()
return
}
resp.Body.Close()
fingerprints := s.wappalyzer.Fingerprint(resp.Header, data)
items := make([]string, 0, len(fingerprints))
for k := range fingerprints {
items = append(items, strings.ToLower(k))
}
hostTagsMappings[value] = items
go func(input string) {
defer inputPool.WaitGroup.Done()
s.processWappalyzerInputPair(input)
}(value)
})
return hostTagsMappings, nil
}
// executeDiscoveredTagsOnTemplates takes a list of hosts and tags and runs templates
// that match these unique tags in directories other than technologies/panels/workflows.
func (s *Service) executeDiscoveredHostTags(data map[string][]string) error {
gologger.Info().Msgf("Executing %d discovered host->tech mappings", len(data))
var allTemplates []string
// Collect path for default directories we want to look for templates in
for _, directory := range defaultTemplatesDirectories {
templates, err := s.opts.Catalog.GetTemplatePath(directory)
if err != nil {
return errors.Wrap(err, "could not get templates in directory")
}
allTemplates = append(allTemplates, templates...)
}
childExecuter := s.engine.ChildExecuter()
for k, v := range data {
templates := s.store.LoadTemplatesWithTags(allTemplates, v)
gologger.Info().Msgf("Executing tags %v for host %s (%d templates)", v, k, len(templates))
for _, template := range templates {
childExecuter.Execute(template, k)
}
}
results := childExecuter.Close()
if results.Load() {
s.results = true
}
inputPool.WaitGroup.Wait()
return nil
}
func (s *Service) processWappalyzerInputPair(input string) {
req, err := retryablehttp.NewRequest(http.MethodGet, input, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", uarand.GetRandom())
resp, err := s.httpclient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
return
}
reader := io.LimitReader(resp.Body, maxDefaultBody)
data, err := ioutil.ReadAll(reader)
if err != nil {
resp.Body.Close()
return
}
resp.Body.Close()
fingerprints := s.wappalyzer.Fingerprint(resp.Header, data)
items := make([]string, 0, len(fingerprints))
for k := range fingerprints {
items = append(items, strings.ToLower(k))
}
if len(items) == 0 {
return
}
templates := s.store.LoadTemplatesWithTags(s.allTemplates, items)
gologger.Info().Msgf("Executing tags %v for host %s (%d templates)", strings.Join(items, ","), input, len(templates))
for _, template := range templates {
s.childExecuter.Execute(template, input)
}
}