External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-18 04:45:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into issue-tracker-integration

Browse Source
This commit is contained in:
Ice3man 2021-02-08 01:51:37 +05:30 committed by GitHub
commit 8b8adb7b46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 3199 additions and 521 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/cmd/nuclei/main.go
View File

@ -73,9 +73,11 @@ based on templates offering massive extensibility and ease of use.`)
set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled")
set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
set.StringVarP(&options.ReportingConfig, "reporting-config", "rc", "", "Nuclei Reporting Module configuration file")
set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database")
set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
_ = set.Parse()
if cfgFile != "" {

9
v2/go.mod
View File

@ -8,7 +8,6 @@ require (
github.com/blang/semver v3.5.1+incompatible
github.com/corpix/uarand v0.1.1
github.com/golang/protobuf v1.4.3 // indirect
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-cmp v0.5.2 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-github/v32 v32.1.0
@ -18,19 +17,19 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible
github.com/mattn/go-runewidth v0.0.10 // indirect
github.com/miekg/dns v1.1.37
github.com/miekg/dns v1.1.38
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/olekukonko/tablewriter v0.0.4
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.7
github.com/projectdiscovery/collaborator v0.0.2
github.com/projectdiscovery/fastdialer v0.0.2
github.com/projectdiscovery/fastdialer v0.0.3
github.com/projectdiscovery/goflags v0.0.2
github.com/projectdiscovery/gologger v1.1.3
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/rawhttp v0.0.4
github.com/projectdiscovery/retryabledns v1.0.5
github.com/projectdiscovery/retryabledns v1.0.6
github.com/projectdiscovery/retryablehttp-go v1.0.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rivo/uniseg v0.2.0 // indirect
@ -43,11 +42,9 @@ require (
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.6.0
go.uber.org/ratelimit v0.1.0
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.25.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect

9
v2/go.sum
View File

@ -68,7 +68,8 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.37/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw=
github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -88,13 +89,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
github.com/projectdiscovery/fastdialer v0.0.3 h1:KYj1DmOEl5ogDuRu0ny8DeDFhGS+W0+sQw1qc7otVSw=
github.com/projectdiscovery/fastdialer v0.0.3/go.mod h1:RtocEuqSbT74NRi+wJI1/lz7da3ncayzbzetrgd3QaQ=
github.com/projectdiscovery/goflags v0.0.2 h1:4vB5+mA41xgW6V1y4YD1A+iI8Kq68iTTny50XuSYKdo=
github.com/projectdiscovery/goflags v0.0.2/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryabledns v1.0.6 h1:fz33puVeUKJJ5s2POSlxO4WA4iodW6Yzm/EVNuO/93w=
github.com/projectdiscovery/retryabledns v1.0.6/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=

6
v2/internal/runner/options.go
View File

@ -70,13 +70,9 @@ func validateOptions(options *types.Options) error {
if !options.TemplateList {
// Check if a list of templates was provided and it exists
if len(options.Templates) == 0 && !options.UpdateTemplates {
if len(options.Templates) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
if options.Targets == "" && !options.Stdin && options.Target == "" && !options.UpdateTemplates {
return errors.New("no target input provided")
}
}
// Validate proxy options if provided

2
v2/internal/runner/processor.go
View File

@ -15,7 +15,6 @@ import (
func (r *Runner) processTemplateWithList(template *templates.Template) bool {
results := &atomic.Bool{}
wg := sizedwaitgroup.New(r.options.BulkSize)
r.hostMap.Scan(func(k, _ []byte) error {
URL := string(k)
@ -32,7 +31,6 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool {
return nil
})
wg.Wait()
return results.Load()
}

61
v2/internal/runner/runner.go
View File

@ -188,6 +188,9 @@ func (r *Runner) Close() {
// binary and runs the actual enumeration
func (r *Runner) RunEnumeration() {
// resolves input templates definitions and any optional exclusion
if len(r.options.Templates) == 0 && len(r.options.Tags) > 0 {
r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
}
includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates)
excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates)
// defaults to all templates
@ -210,15 +213,6 @@ func (r *Runner) RunEnumeration() {
}
}
executerOpts := &protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
}
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
@ -237,18 +231,28 @@ func (r *Runner) RunEnumeration() {
clusterCount := 0
clusters := clusterer.Cluster(availableTemplates)
for _, cluster := range clusters {
if len(cluster) > 1 {
if len(cluster) > 1 && !r.options.OfflineHTTP {
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
}
clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
finalTemplates = append(finalTemplates, &templates.Template{
ID: clusterID,
RequestsHTTP: cluster[0].RequestsHTTP,
Executer: clusterer.NewExecuter(cluster, executerOpts),
Executer: clusterer.NewExecuter(cluster, &executerOpts),
TotalRequests: len(cluster[0].RequestsHTTP),
})
clusterCount++
} else {
finalTemplates = append(finalTemplates, cluster[0])
for _, item := range cluster {
finalTemplates = append(finalTemplates, item)
}
}
}
@ -263,7 +267,6 @@ func (r *Runner) RunEnumeration() {
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
}
templateCount := originalTemplatesCount
hasWorkflows := workflowCount > 0
// 0 matches means no templates were found in directory
if templateCount == 0 {
@ -280,27 +283,23 @@ func (r *Runner) RunEnumeration() {
// Starts polling or ignore
collaborator.DefaultCollaborator.Poll()
if r.inputCount == 0 {
gologger.Error().Msgf("Could not find any valid input URLs.")
} else if totalRequests > 0 || hasWorkflows {
// tracks global progress and captures stdout/stderr until p.Wait finishes
r.progress.Init(r.inputCount, templateCount, totalRequests)
// tracks global progress and captures stdout/stderr until p.Wait finishes
r.progress.Init(r.inputCount, templateCount, totalRequests)
for _, t := range finalTemplates {
wgtemplates.Add()
go func(template *templates.Template) {
defer wgtemplates.Done()
for _, t := range finalTemplates {
wgtemplates.Add()
go func(template *templates.Template) {
defer wgtemplates.Done()
if len(template.Workflows) > 0 {
results.CAS(false, r.processWorkflowWithList(template))
} else {
results.CAS(false, r.processTemplateWithList(template))
}
}(t)
}
wgtemplates.Wait()
r.progress.Stop()
if len(template.Workflows) > 0 {
results.CAS(false, r.processWorkflowWithList(template))
} else {
results.CAS(false, r.processTemplateWithList(template))
}
}(t)
}
wgtemplates.Wait()
r.progress.Stop()
if r.issuesClient != nil {
r.issuesClient.Close()

24
v2/internal/runner/templates.go
View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
@ -23,16 +24,16 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri
for _, match := range templatePaths {
t, err := r.parseTemplateFile(match)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", match, err)
gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
continue
}
if len(t.Workflows) > 0 {
workflowCount++
}
sev := strings.ToLower(t.Info["severity"])
sev := strings.ToLower(types.ToString(t.Info["severity"]))
if !filterBySeverity || hasMatchingSeverity(sev, severities) {
parsedTemplates[t.ID] = t
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"]))
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
} else {
gologger.Error().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
}
@ -42,14 +43,13 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
executerOpts := &protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
IssuesClient: r.issuesClient,
ProjectFile: r.projectFile,
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
}
template, err := templates.Parse(file, executerOpts)
if err != nil {
@ -75,7 +75,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) {
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"]))
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
}
}

118
v2/internal/testutils/testutils.go Normal file
View File

@ -0,0 +1,118 @@
package testutils
import (
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/catalogue"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"go.uber.org/ratelimit"
)
// Init initializes the protocols and their configurations
func Init(options *types.Options) {
protocolinit.Init(options)
}
// DefaultOptions is the default options structure for nuclei during mocking.
var DefaultOptions = &types.Options{
RandomAgent: false,
Metrics: false,
Debug: false,
DebugRequests: false,
DebugResponse: false,
Silent: false,
Version: false,
Verbose: false,
NoColor: true,
UpdateTemplates: false,
JSON: false,
JSONRequests: false,
EnableProgressBar: false,
TemplatesVersion: false,
TemplateList: false,
Stdin: false,
StopAtFirstMatch: false,
NoMeta: false,
Project: false,
MetricsPort: 0,
BulkSize: 25,
TemplateThreads: 10,
Timeout: 5,
Retries: 1,
RateLimit: 150,
BurpCollaboratorBiid: "",
ProjectPath: "",
Severity: []string{},
Target: "",
Targets: "",
Output: "",
ProxyURL: "",
ProxySocksURL: "",
TemplatesDirectory: "",
TraceLogFile: "",
Templates: []string{},
ExcludedTemplates: []string{},
CustomHeaders: []string{},
}
// MockOutputWriter is a mocked output writer.
type MockOutputWriter struct {
aurora aurora.Aurora
RequestCallback func(templateID, url, requestType string, err error)
WriteCallback func(o *output.ResultEvent)
}
// NewMockOutputWriter creates a new mock output writer
func NewMockOutputWriter() *MockOutputWriter {
return &MockOutputWriter{aurora: aurora.NewAurora(false)}
}
// Close closes the output writer interface
func (m *MockOutputWriter) Close() {}
// Colorizer returns the colorizer instance for writer
func (m *MockOutputWriter) Colorizer() aurora.Aurora {
return m.aurora
}
// Write writes the event to file and/or screen.
func (m *MockOutputWriter) Write(result *output.ResultEvent) error {
if m.WriteCallback != nil {
m.WriteCallback(result)
}
return nil
}
// Request writes a log the requests trace log
func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) {
if m.RequestCallback != nil {
m.RequestCallback(templateID, url, requestType, err)
}
}
// TemplateInfo contains info for a mock executed template.
type TemplateInfo struct {
ID string
Info map[string]interface{}
Path string
}
// NewMockExecuterOptions creates a new mock executeroptions struct
func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecuterOptions {
progress, _ := progress.NewProgress(false, false, 0)
executerOpts := &protocols.ExecuterOptions{
TemplateID: info.ID,
TemplateInfo: info.Info,
TemplatePath: info.Path,
Output: NewMockOutputWriter(),
Options: options,
Progress: progress,
ProjectFile: nil,
Catalogue: catalogue.New(options.TemplatesDirectory),
RateLimiter: ratelimit.New(options.RateLimit),
}
return executerOpts
}

1
v2/pkg/operators/extractors/compile.go
View File

@ -21,7 +21,6 @@ func (e *Extractor) CompileExtractors() error {
if err != nil {
return fmt.Errorf("could not compile regex: %s", regex)
}
e.regexCompiled = append(e.regexCompiled, compiled)
}

2
v2/pkg/output/format_screen.go
View File

@ -27,7 +27,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) ([]byte, error) {
builder.WriteString("] ")
builder.WriteString("[")
builder.WriteString(w.severityColors.Data[output.Info["severity"]])
builder.WriteString(w.severityColors.Data[types.ToString(output.Info["severity"])])
builder.WriteString("] ")
}
builder.WriteString(output.Matched)

8
v2/pkg/output/output.go
View File

@ -21,7 +21,7 @@ type Writer interface {
Colorizer() aurora.Aurora
// Write writes the event to file and/or screen.
Write(*ResultEvent) error
// Request writes a log the requests trace log
// Request logs a request in the trace log
Request(templateID, url, requestType string, err error)
}
@ -54,7 +54,7 @@ type ResultEvent struct {
// TemplateID is the ID of the template for the result.
TemplateID string `json:"templateID"`
// Info contains information block of the template for the result.
Info map[string]string `json:"info,inline"`
Info map[string]interface{} `json:"info,inline"`
// MatcherName is the name of the matcher matched if any.
MatcherName string `json:"matcher_name,omitempty"`
// ExtractorName is the name of the extractor matched if any.
@ -125,6 +125,9 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
if err != nil {
return errors.Wrap(err, "could not format output")
}
if len(data) == 0 {
return nil
}
_, _ = os.Stdout.Write(data)
_, _ = os.Stdout.Write([]byte("\n"))
if w.outputFile != nil {
@ -134,7 +137,6 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
if writeErr := w.outputFile.Write(data); writeErr != nil {
return errors.Wrap(err, "could not write to output")
}
_ = w.outputFile.Write([]byte("\n"))
}
return nil
}

2
v2/pkg/protocols/common/clusterer/clusterer_test.go
View File

@ -23,7 +23,7 @@ func TestHTTPRequestsCluster(t *testing.T) {
protocolinit.Init(&types.Options{})
list := make(map[string]*templates.Template)
for _, template := range templatesList {
executerOpts := &protocols.ExecuterOptions{
executerOpts := protocols.ExecuterOptions{
Output: &mockOutput{},
Options: &types.Options{},
Progress: nil,

2
v2/pkg/protocols/common/clusterer/executer.go
View File

@ -22,7 +22,7 @@ type Executer struct {
type clusteredOperator struct {
templateID string
templateInfo map[string]string
templateInfo map[string]interface{}
operator *operators.Operators
}

25
v2/pkg/protocols/common/replacer/replacer.go
View File

@ -1,8 +1,7 @@
package replacer
import (
"fmt"
"strings"
"github.com/valyala/fasttemplate"
)
// Payload marker constants
@ -12,21 +11,9 @@ const (
MarkerParenthesisClose = "}}"
)
// New creates a new replacer structure for values replacement on the fly.
func New(values map[string]interface{}) *strings.Replacer {
replacerItems := make([]string, 0, len(values)*4)
for key, val := range values {
valueStr := fmt.Sprintf("%s", val)
replacerItems = append(replacerItems,
fmt.Sprintf("%s%s%s", MarkerParenthesisOpen, key, MarkerParenthesisClose),
valueStr,
)
replacerItems = append(replacerItems,
fmt.Sprintf("%s%s%s", MarkerGeneral, key, MarkerGeneral),
valueStr,
)
}
return strings.NewReplacer(replacerItems...)
// Replace replaces placeholders in template with values on the fly.
func Replace(template string, values map[string]interface{}) string {
new := fasttemplate.ExecuteStringStd(template, MarkerGeneral, MarkerGeneral, values)
final := fasttemplate.ExecuteStringStd(new, MarkerParenthesisOpen, MarkerParenthesisClose, values)
return final
}

6
v2/pkg/protocols/dns/dns.go
View File

@ -26,8 +26,6 @@ type Request struct {
Class string `yaml:"class"`
// Retries is the number of retries for the DNS request
Retries int `yaml:"retries"`
// Raw contains a raw request
Raw string `yaml:"raw,omitempty"`
// Operators for the current request go here.
operators.Operators `yaml:",inline"`
@ -85,9 +83,9 @@ func (r *Request) Make(domain string) (*dns.Msg, error) {
var q dns.Question
replacer := replacer.New(map[string]interface{}{"FQDN": domain})
final := replacer.Replace(r.Name, map[string]interface{}{"FQDN": domain})
q.Name = dns.Fqdn(replacer.Replace(r.Name))
q.Name = dns.Fqdn(final)
q.Qclass = r.class
q.Qtype = r.question
req.Question = append(req.Question, q)

33
v2/pkg/protocols/dns/dns_test.go Normal file
View File

@ -0,0 +1,33 @@
package dns
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestDNSCompileMake(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
req, err := request.Make("one.one.one.one")
require.Nil(t, err, "could not make dns request")
require.Equal(t, "one.one.one.one.", req.Question[0].Name, "could not get correct dns question")
}

47
v2/pkg/protocols/dns/operators.go
View File

@ -23,23 +23,18 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
if !ok {
return false
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["rcode"]
if !ok {
return false
}
return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
return matcher.Result(matcher.MatchStatusCode(item.(int)))
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(types.ToString(item))))
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr))
return matcher.Result(matcher.MatchWords(types.ToString(item)))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.Result(matcher.MatchRegex(types.ToString(item)))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.Result(matcher.MatchBinary(types.ToString(item)))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
}
@ -48,18 +43,13 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part, ok := data[extractor.Part]
if !ok {
return nil
}
partString := part.(string)
switch partString {
part := extractor.Part
switch part {
case "body", "all":
partString = "raw"
part = "raw"
}
item, ok := data[partString]
item, ok := data[part]
if !ok {
return nil
}
@ -76,15 +66,12 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 8)
data := make(output.InternalEvent, 11)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
if r.options.Options.JSONRequests {
data["request"] = req.String()
}
data["request"] = req.String()
data["rcode"] = resp.Rcode
buffer := &bytes.Buffer{}
@ -149,17 +136,17 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: wrapped.InternalEvent["template-id"].(string),
Info: wrapped.InternalEvent["template-info"].(map[string]string),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "dns",
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)
data.Response = wrapped.InternalEvent["raw"].(string)
data.Request = types.ToString(wrapped.InternalEvent["request"])
data.Response = types.ToString(wrapped.InternalEvent["raw"])
}
return data
}

241
v2/pkg/protocols/dns/operators_test.go Normal file
View File

@ -0,0 +1,241 @@
package dns
import (
"net"
"strconv"
"testing"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
req := new(dns.Msg)
req.Question = append(req.Question, dns.Question{Name: "one.one.one.one.", Qtype: dns.TypeA, Qclass: dns.ClassINET})
resp := new(dns.Msg)
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 11, "could not get correct number of items in dsl map")
require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode")
}
func TestDNSOperatorMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
req := new(dns.Msg)
req.Question = append(req.Question, dns.Question{Name: "one.one.one.one.", Qtype: dns.TypeA, Qclass: dns.ClassINET})
resp := new(dns.Msg)
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
})
t.Run("rcode", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "rcode",
Type: "status",
Status: []int{dns.RcodeSuccess},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile rcode matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid rcode response")
})
t.Run("negative", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Negative: true,
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
})
t.Run("invalid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
})
}
func TestDNSOperatorExtract(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
req := new(dns.Msg)
req.Question = append(req.Question, dns.Question{Name: "one.one.one.one.", Qtype: dns.TypeA, Qclass: dns.ClassINET})
resp := new(dns.Msg)
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor valid response")
require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
})
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
KVal: []string{"rcode"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile kval extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor kval valid response")
require.Equal(t, map[string]struct{}{strconv.Itoa(dns.RcodeSuccess): {}}, data, "could not extract correct kval data")
})
}
func TestDNSMakeResult(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
req := new(dns.Msg)
req.Question = append(req.Question, dns.Question{Name: "one.one.one.one.", Qtype: dns.TypeA, Qclass: dns.ClassINET})
resp := new(dns.Msg)
resp.Rcode = dns.RcodeSuccess
resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}})
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}

77
v2/pkg/protocols/dns/request_test.go Normal file
View File

@ -0,0 +1,77 @@
package dns
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestDNSExecuteWithResults(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-dns"
request := &Request{
Type: "A",
Class: "INET",
Retries: 5,
ID: templateID,
Recursion: false,
Name: "{{FQDN}}",
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"93.184.216.34"},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile dns request")
var finalEvent *output.InternalWrappedEvent
t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute dns request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
t.Run("url-to-domain", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute dns request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
}

5
v2/pkg/protocols/file/file.go
View File

@ -18,8 +18,6 @@ type Request struct {
NoRecursive bool `yaml:"no-recursive"`
// Extensions is the list of extensions to perform matching on.
Extensions []string `yaml:"extensions"`
// ExtensionAllowlist is the list of file extensions to enforce allowing.
ExtensionAllowlist []string `yaml:"allowlist"`
// ExtensionDenylist is the list of file extensions to deny during matching.
ExtensionDenylist []string `yaml:"denylist"`
@ -73,9 +71,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
for _, extension := range r.ExtensionDenylist {
r.extensionDenylist[extension] = struct{}{}
}
for _, extension := range r.ExtensionAllowlist {
delete(r.extensionDenylist, extension)
}
return nil
}

32
v2/pkg/protocols/file/file_test.go Normal file
View File

@ -0,0 +1,32 @@
package file
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestFileCompile(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist")
require.NotContains(t, request.extensions, ".go", "could get .go in allowlist")
require.True(t, request.allExtensions, "could not get correct allExtensions")
}

7
v2/pkg/protocols/file/find.go
View File

@ -87,6 +87,9 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
return godirwalk.SkipNode
},
Callback: func(path string, d *godirwalk.Dirent) error {
if d.IsDir() {
return nil
}
if !r.validatePath(path) {
return nil
}
@ -103,11 +106,11 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
// validatePath validates a file path for blacklist and whitelist options
func (r *Request) validatePath(item string) bool {
extension := path.Ext(item)
if len(r.extensions) > 0 && !r.allExtensions {
if len(r.extensions) > 0 {
if _, ok := r.extensions[extension]; ok {
return true
}
return false
}
if _, ok := r.extensionDenylist[extension]; ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)

63
v2/pkg/protocols/file/find_test.go Normal file
View File

@ -0,0 +1,63 @@
package file
import (
"io/ioutil"
"os"
"path"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestFindInputPaths(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
tempDir, err := ioutil.TempDir("", "test-*")
require.Nil(t, err, "could not create temporary directory")
defer os.RemoveAll(tempDir)
files := map[string]string{
"test.go": "TEST",
"config.yaml": "TEST",
"final.yaml": "TEST",
"image_ignored.png": "TEST",
"test.js": "TEST",
}
for k, v := range files {
err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
require.Nil(t, err, "could not write temporary file")
}
expected := []string{"config.yaml", "final.yaml", "test.js"}
got := []string{}
err = request.getInputPaths(tempDir+"/*", func(item string) {
base := path.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for glob")
require.ElementsMatch(t, expected, got, "could not get correct file matches for glob")
got = []string{}
err = request.getInputPaths(tempDir, func(item string) {
base := path.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for directory")
require.ElementsMatch(t, expected, got, "could not get correct file matches for directory")
}

23
v2/pkg/protocols/file/operators.go
View File

@ -13,7 +13,7 @@ import (
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
partString := matcher.Part
switch partString {
case "body", "all", "":
case "body", "all", "data", "":
partString = "raw"
}
@ -40,14 +40,9 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part, ok := data[extractor.Part]
if !ok {
return nil
}
partString := part.(string)
partString := extractor.Part
switch partString {
case "body", "all", "":
case "body", "all", "data", "":
partString = "raw"
}
@ -68,7 +63,7 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(raw string, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 3)
data := make(output.InternalEvent, 5)
// Some data regarding the request metadata
data["host"] = host
@ -109,16 +104,16 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: wrapped.InternalEvent["template-id"].(string),
Info: wrapped.InternalEvent["template-info"].(map[string]string),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "file",
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Response = wrapped.InternalEvent["raw"].(string)
data.Response = types.ToString(wrapped.InternalEvent["raw"])
}
return data
}

204
v2/pkg/protocols/file/operators_test.go Normal file
View File

@ -0,0 +1,204 @@
package file
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 5, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
}
func TestFileOperatorMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 5, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
})
t.Run("negative", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Negative: true,
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
})
t.Run("invalid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
})
}
func TestFileOperatorExtract(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 5, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor valid response")
require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
})
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
KVal: []string{"raw"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile kval extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor kval valid response")
require.Equal(t, map[string]struct{}{resp: {}}, data, "could not extract correct kval data")
})
}
func TestFileMakeResult(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 5, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}

1
v2/pkg/protocols/file/request.go
View File

@ -47,7 +47,6 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int
return
}
dataStr := tostring.UnsafeToString(buffer)
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data)
gologger.Print().Msgf("%s", dataStr)

76
v2/pkg/protocols/file/request_test.go Normal file
View File

@ -0,0 +1,76 @@
package file
import (
"io/ioutil"
"os"
"path"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestFileExecuteWithResults(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-file"
request := &Request{
ID: templateID,
MaxSize: 1024,
NoRecursive: false,
Extensions: []string{"*"},
ExtensionDenylist: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
tempDir, err := ioutil.TempDir("", "test-*")
require.Nil(t, err, "could not create temporary directory")
defer os.RemoveAll(tempDir)
files := map[string]string{
"config.yaml": "TEST\r\n1.1.1.1\r\n",
}
for k, v := range files {
err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
require.Nil(t, err, "could not write temporary file")
}
var finalEvent *output.InternalWrappedEvent
t.Run("valid", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults(tempDir, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute file request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
}

206
v2/pkg/protocols/http/build_request.go
View File

@ -2,7 +2,6 @@ package http
import (
"context"
"fmt"
"io"
"io/ioutil"
"net"
@ -27,72 +26,6 @@ var (
templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
)
// requestGenerator generates requests sequentially based on various
// configurations for a http request template.
//
// If payload values are present, an iterator is created for the payload
// values. Paths and Raw requests are supported as base input, so
// it will automatically select between them based on the template.
type requestGenerator struct {
currentIndex int
request *Request
payloadIterator *generators.Iterator
}
// newGenerator creates a new request generator instance
func (r *Request) newGenerator() *requestGenerator {
generator := &requestGenerator{request: r}
if len(r.Payloads) > 0 {
generator.payloadIterator = r.generator.NewIterator()
}
return generator
}
// nextValue returns the next path or the next raw request depending on user input
// It returns false if all the inputs have been exhausted by the generator instance.
func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) {
// If we have paths, return the next path.
if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) {
if item := r.request.Path[r.currentIndex]; item != "" {
r.currentIndex++
return item, nil, true
}
}
// If we have raw requests, start with the request at current index.
// If we are not at the start, then check if the iterator for payloads
// has finished if there are any.
//
// If the iterator has finished for the current raw request
// then reset it and move on to the next value, otherwise use the last request.
if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) {
if r.payloadIterator != nil {
payload, ok := r.payloadIterator.Value()
if !ok {
r.currentIndex++
r.payloadIterator.Reset()
// No more payloads request for us now.
if len(r.request.Raw) == r.currentIndex {
return "", nil, false
}
if item := r.request.Raw[r.currentIndex]; item != "" {
newPayload, ok := r.payloadIterator.Value()
return item, newPayload, ok
}
return "", nil, false
}
return r.request.Raw[r.currentIndex], payload, true
}
if item := r.request.Raw[r.currentIndex]; item != "" {
r.currentIndex++
return item, nil, true
}
}
return "", nil, false
}
// generatedRequest is a single wrapped generated request for a template request
type generatedRequest struct {
original *Request
@ -105,8 +38,7 @@ type generatedRequest struct {
// Make creates a http request for the provided input.
// It returns io.EOF as error when all the requests have been exhausted.
func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) {
baseURL = strings.TrimSuffix(baseURL, "/")
// We get the next payload for the request.
data, payloads, ok := r.nextValue()
if !ok {
return nil, io.EOF
@ -118,16 +50,22 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
return nil, err
}
hostname := parsed.Host
data, parsed = baseURLWithTemplatePrefs(data, parsed)
values := generators.MergeMaps(dynamicValues, map[string]interface{}{
"BaseURL": baseURLWithTemplatePrefs(data, parsed),
"Hostname": hostname,
"Hostname": parsed.Hostname(),
})
// If data contains \n it's a raw request, process it like that. Else
isRawRequest := strings.Contains(data, "\n")
if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") {
parsed.Path = strings.TrimSuffix(parsed.Path, "/")
}
parsedString := parsed.String()
values["BaseURL"] = parsedString
// If data contains \n it's a raw request, process it like raw. Else
// continue with the template based request flow.
if strings.Contains(data, "\n") {
return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values, payloads)
if isRawRequest {
return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads)
}
return r.makeHTTPRequestFromModel(ctx, data, values)
}
@ -141,26 +79,28 @@ func (r *requestGenerator) Total() int {
}
// baseURLWithTemplatePrefs returns the url for BaseURL keeping
// the template port and path preference
func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string {
// template port preference over input URL port
// template has port
hasPort := len(urlWithPortRegex.FindStringSubmatch(data)) > 0
if hasPort {
// check if also the input contains port, in this case extracts the url
if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil {
parsedURL.Host = hostname
}
// the template port and path preference over the user provided one.
func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
// template port preference over input URL port if template has a port
matches := urlWithPortRegex.FindAllStringSubmatch(data, -1)
if len(matches) == 0 {
return data, parsed
}
return parsedURL.String()
port := matches[0][1]
parsed.Host = net.JoinHostPort(parsed.Hostname(), port)
data = strings.ReplaceAll(data, ":"+port, "")
if parsed.Path == "" {
parsed.Path = "/"
}
return data, parsed
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*generatedRequest, error) {
URL := replacer.New(values).Replace(data)
final := replacer.Replace(data, values)
// Build a request on the specified URL
req, err := http.NewRequestWithContext(ctx, r.request.Method, URL, nil)
req, err := http.NewRequestWithContext(ctx, r.request.Method, final, nil)
if err != nil {
return nil, err
}
@ -174,32 +114,27 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) {
// Add trailing line
data += "\n"
// If we have payloads, handle them by evaluating them at runtime.
if len(r.request.Payloads) > 0 {
finalPayloads, err := r.getPayloadValues(baseURL, payloads)
if err != nil {
return nil, err
}
return r.handleRawWithPaylods(ctx, data, baseURL, values, finalPayloads)
}
return r.handleRawWithPaylods(ctx, data, baseURL, values, nil)
data += "\r\n"
return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
}
// handleRawWithPaylods handles raw requests along with paylaods
func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) {
baseValues := generators.CopyMap(values)
finalValues := generators.MergeMaps(baseValues, generatorValues)
// Replace the dynamic variables in the URL if any
rawRequest = replacer.New(finalValues).Replace(rawRequest)
// Combine the template payloads along with base
// request values.
finalValues := generators.MergeMaps(generatorValues, values)
rawRequest = replacer.Replace(rawRequest, finalValues)
// Check if the match contains a dynamic variable, for each
// found one we will check if it's an expression and can
// be compiled, it will be evaluated and the results will be returned.
//
// The provided keys from finalValues will be used as variable names
// for substitution inside the expression.
dynamicValues := make(map[string]interface{})
for _, match := range templateExpressionRegex.FindAllString(rawRequest, -1) {
// check if the match contains a dynamic variable
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions())
if err != nil {
return nil, err
@ -208,17 +143,17 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
if err != nil {
return nil, err
}
dynamicValues[expr] = result
dynamicValues[expr] = result // convert base64(<payload_name>) => <base64-representation>
}
// Replacer dynamic values if any in raw request and parse it
rawRequest = replacer.New(dynamicValues).Replace(rawRequest)
rawRequest = replacer.Replace(rawRequest, dynamicValues)
rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe)
if err != nil {
return nil, err
}
// rawhttp
// Unsafe option uses rawhttp library
if r.request.Unsafe {
unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request}
return unsafeReq, nil
@ -237,12 +172,9 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
if err != nil {
return nil, err
}
// copy headers
for key, value := range rawRequestData.Headers {
req.Header[key] = []string{value}
}
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
@ -253,9 +185,8 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest,
// fillRequest fills various headers in the request with values
func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
// Set the header values requested
replacer := replacer.New(values)
for header, value := range r.request.Headers {
req.Header[header] = []string{replacer.Replace(value)}
req.Header[header] = []string{replacer.Replace(value, values)}
}
// In case of multiple threads the underlying connection should remain open to allow reuse
@ -269,13 +200,11 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte
}
setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
// raw requests are left untouched
if len(r.request.Raw) > 0 {
return retryablehttp.FromRequest(req)
// Only set these headers on non raw requests
if len(r.request.Raw) == 0 {
setHeader(req, "Accept", "*/*")
setHeader(req, "Accept-Language", "en")
}
setHeader(req, "Accept", "*/*")
setHeader(req, "Accept-Language", "en")
return retryablehttp.FromRequest(req)
}
@ -285,40 +214,3 @@ func setHeader(req *http.Request, name, value string) {
req.Header.Set(name, value)
}
}
// getPayloadValues returns current payload values for a request
func (r *requestGenerator) getPayloadValues(reqURL string, templatePayloads map[string]interface{}) (map[string]interface{}, error) {
payloadProcessedValues := make(map[string]interface{})
for k, v := range templatePayloads {
kexp := v.(string)
// if it doesn't containing markups, we just continue
if !strings.Contains(kexp, replacer.MarkerParenthesisOpen) || strings.Contains(kexp, replacer.MarkerParenthesisClose) || strings.Contains(kexp, replacer.MarkerGeneral) {
payloadProcessedValues[k] = v
continue
}
// attempts to expand expressions
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, dsl.HelperFunctions())
if err != nil {
// it is a simple literal payload => proceed with literal value
payloadProcessedValues[k] = v
continue
}
// it is an expression - try to solve it
expValue, err := compiled.Evaluate(templatePayloads)
if err != nil {
// an error occurred => proceed with literal value
payloadProcessedValues[k] = v
continue
}
payloadProcessedValues[k] = fmt.Sprint(expValue)
}
var err error
if len(payloadProcessedValues) == 0 {
err = ErrNoPayload
}
return payloadProcessedValues, err
}
// ErrNoPayload error to avoid the additional base null request
var ErrNoPayload = fmt.Errorf("no payload found")

190
v2/pkg/protocols/http/build_request_test.go
View File

@ -1,70 +1,158 @@
package http
import (
"net/url"
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestRequestGeneratorPaths(t *testing.T) {
req := &Request{
Path: []string{"{{BaseURL}}/test", "{{BaseURL}}/test.php"},
}
generator := req.newGenerator()
var payloads []string
for {
raw, _, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, raw)
}
require.Equal(t, req.Path, payloads, "Could not get correct paths")
func TestBaseURLWithTemplatePrefs(t *testing.T) {
baseURL := "http://localhost:53/test"
parsed, _ := url.Parse(baseURL)
data := "{{BaseURL}}:8000/newpath"
data, parsed = baseURLWithTemplatePrefs(data, parsed)
require.Equal(t, "http://localhost:8000/test", parsed.String(), "could not get correct value")
require.Equal(t, "{{BaseURL}}/newpath", data, "could not get correct data")
}
func TestRequestGeneratorClusterSingle(t *testing.T) {
var err error
func TestMakeRequestFromModal(t *testing.T) {
options := testutils.DefaultOptions
req := &Request{
Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}},
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`},
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}/login.php"},
Method: "POST",
Body: "username=test&password=pass",
Headers: map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": "1",
},
}
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
generator := req.newGenerator()
var payloads []map[string]interface{}
for {
_, data, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, data)
}
require.Equal(t, 9, len(payloads), "Could not get correct number of payloads")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
bodyBytes, _ := req.request.BodyBytes()
require.Equal(t, "/login.php", req.request.URL.Path, "could not get correct request path")
require.Equal(t, "username=test&password=pass", string(bodyBytes), "could not get correct request body")
}
func TestRequestGeneratorClusterMultipleRaw(t *testing.T) {
var err error
func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
options := testutils.DefaultOptions
req := &Request{
Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}},
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?query=example"},
Method: "GET",
}
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
generator := req.newGenerator()
var payloads []map[string]interface{}
for {
_, data, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, data)
}
require.Equal(t, 18, len(payloads), "Could not get correct number of payloads")
generator := request.newGenerator()
req, err := generator.Make("https://example.com/test.php", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
generator = request.newGenerator()
req, err = generator.Make("https://example.com/test/", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
}
func TestMakeRequestFromRawWithPayloads(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Payloads: map[string]interface{}{
"username": []string{"admin"},
"password": []string{"admin", "guest", "password", "test", "12345", "123456"},
},
AttackType: "clusterbomb",
Raw: []string{`GET /manager/html HTTP/1.1
Host: {{Hostname}}
User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei)
Connection: close
Authorization: Basic {{username + ':' + password}}
Accept-Encoding: gzip`},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization := req.request.Header.Get("Authorization")
require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
req, err = generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization = req.request.Header.Get("Authorization")
require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
}
func TestMakeRequestFromRawPayloadExpressions(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Payloads: map[string]interface{}{
"username": []string{"admin"},
"password": []string{"admin", "guest", "password", "test", "12345", "123456"},
},
AttackType: "clusterbomb",
Raw: []string{`GET /manager/html HTTP/1.1
Host: {{Hostname}}
User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei)
Connection: close
Authorization: Basic {{base64(username + ':' + password)}}
Accept-Encoding: gzip`},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
generator := request.newGenerator()
req, err := generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization := req.request.Header.Get("Authorization")
require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
req, err = generator.Make("https://example.com", map[string]interface{}{})
require.Nil(t, err, "could not make http request")
authorization = req.request.Header.Get("Authorization")
require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
}

15
v2/pkg/protocols/http/cluster_test.go Normal file
View File

@ -0,0 +1,15 @@
package http
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestCanCluster(t *testing.T) {
req := &Request{Unsafe: true}
require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request")
req = &Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}
require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}), "could not cluster GET request")
}

26
v2/pkg/protocols/http/http.go
View File

@ -57,8 +57,10 @@ type Request struct {
// DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests
DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header"`
// Race determines if all the request have to be attempted at the same time
// The minimum number fof requests is determined by threads
// The minimum number of requests is determined by threads
Race bool `yaml:"race"`
// MaxSize is the maximum size of http response body to read in bytes.
MaxSize int `yaml:"max-size"`
// Operators for the current request go here.
operators.Operators `yaml:",inline"`
@ -93,14 +95,22 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
r.httpClient = client
r.options = options
for _, option := range r.options.Options.CustomHeaders {
parts := strings.SplitN(option, ":", 1)
parts := strings.SplitN(option, ":", 2)
if len(parts) != 2 {
continue
}
r.customHeaders[parts[0]] = strings.TrimSpace(parts[1])
}
if r.Body != "" && !strings.Contains(r.Body, "\r\n") {
r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n")
}
if len(r.Raw) > 0 {
for i, raw := range r.Raw {
if !strings.Contains(raw, "\r\n") {
r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n")
}
}
r.rawhttpClient = httpclientpool.GetRawHTTP()
}
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
@ -122,15 +132,11 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
for name, payload := range r.Payloads {
switch pt := payload.(type) {
case string:
elements := strings.Split(pt, "\n")
//golint:gomnd // this is not a magic number
if len(elements) < 2 {
final, err := options.Catalogue.ResolvePath(elements[0], options.TemplatePath)
if err != nil {
return errors.Wrap(err, "could not read payload file")
}
r.Payloads[name] = final
final, err := options.Catalogue.ResolvePath(pt, options.TemplatePath)
if err != nil {
return errors.Wrap(err, "could not read payload file")
}
r.Payloads[name] = final
}
}

39
v2/pkg/protocols/http/http_test.go Normal file
View File

@ -0,0 +1,39 @@
package http
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestHTTPCompile(t *testing.T) {
options := testutils.DefaultOptions
options.CustomHeaders = []string{"User-Agent: test", "Hello: World"}
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Payloads: map[string]interface{}{
"username": []string{"admin"},
"password": []string{"admin", "guest", "password", "test", "12345", "123456"},
},
AttackType: "clusterbomb",
Raw: []string{`GET /manager/html HTTP/1.1
Host: {{Hostname}}
User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei)
Connection: close
Authorization: Basic {{username + ':' + password}}
Accept-Encoding: gzip`},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile http request")
require.Equal(t, 6, request.Requests(), "could not get correct number of requests")
require.Equal(t, map[string]string{"User-Agent": "test", "Hello": "World"}, request.customHeaders, "could not get correct custom headers")
}

23
v2/pkg/protocols/http/operators.go
View File

@ -63,8 +63,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) {
if part == "all" {
builder := &strings.Builder{}
builder.WriteString(data["body"].(string))
builder.WriteString(data["all_headers"].(string))
builder.WriteString(types.ToString(data["body"]))
builder.WriteString(types.ToString(data["all_headers"]))
itemStr = builder.String()
} else {
item, ok := data[part]
@ -86,18 +86,15 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["host"] = host
data["matched"] = matched
data["request"] = rawReq
data["raw"] = rawResp
data["response"] = rawResp
data["content_length"] = resp.ContentLength
data["status_code"] = resp.StatusCode
data["body"] = body
for _, cookie := range resp.Cookies() {
data[strings.ToLower(cookie.Name)] = cookie.Value
}
for k, v := range resp.Header {
k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
k = strings.ToLower(strings.TrimSpace(k))
data[k] = strings.Join(v, " ")
}
data["all_headers"] = headers
@ -137,19 +134,19 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: wrapped.InternalEvent["template-id"].(string),
Info: wrapped.InternalEvent["template-info"].(map[string]string),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
IP: wrapped.InternalEvent["ip"].(string),
Timestamp: time.Now(),
IP: types.ToString(wrapped.InternalEvent["ip"]),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)
data.Response = wrapped.InternalEvent["raw"].(string)
data.Request = types.ToString(wrapped.InternalEvent["request"])
data.Response = types.ToString(wrapped.InternalEvent["raw"])
}
return data
}

307
v2/pkg/protocols/http/operators_test.go Normal file
View File

@ -0,0 +1,307 @@
package http
import (
"net/http"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
}
func TestHTTPOperatorMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Words: []string{"1.1.1.1"},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
})
t.Run("negative", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Negative: true,
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
})
t.Run("invalid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
})
}
func TestHTTPOperatorExtract(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test-Header", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header")
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor valid response")
require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
})
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
KVal: []string{"test-header"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile kval extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor kval valid response")
require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data")
})
}
func TestHTTPMakeResult(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{
ID: templateID,
Name: "testing",
Path: []string{"{{BaseURL}}?test=1"},
Method: "GET",
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "body",
Type: "word",
Words: []string{"1.1.1.1"},
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}
const exampleRawRequest = `GET / HTTP/1.1
Host: example.com
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9,hi;q=0.8
If-None-Match: "3147526947+gzip"
If-Modified-Since: Thu, 17 Oct 2019 07:18:26 GMT
Connection: close
`
const exampleRawResponse = exampleResponseHeader + exampleResponseBody
const exampleResponseHeader = `
HTTP/1.1 200 OK
Accept-Ranges: bytes
Age: 493322
Cache-Control: max-age=604800
Content-Type: text/html; charset=UTF-8
Date: Thu, 04 Feb 2021 12:15:51 GMT
Etag: "3147526947+ident"
Expires: Thu, 11 Feb 2021 12:15:51 GMT
Last-Modified: Thu, 17 Oct 2019 07:18:26 GMT
Server: ECS (nyb/1D1C)
Vary: Accept-Encoding
X-Cache: HIT
Content-Length: 1256
Connection: close
`
const exampleResponseBody = `
<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8" />
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style type="text/css">
body {
background-color: #f0f0f2;
margin: 0;
padding: 0;
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
}
div {
width: 600px;
margin: 5em auto;
padding: 2em;
background-color: #fdfdff;
border-radius: 0.5em;
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
}
a:link, a:visited {
color: #38488f;
text-decoration: none;
}
@media (max-width: 700px) {
div {
margin: 0 auto;
width: auto;
}
}
</style>
</head>
<a>1.1.1.1</a>
<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
`

4
v2/pkg/protocols/http/raw/raw.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"fmt"
"io/ioutil"
"net"
"net/url"
"strings"
)
@ -90,6 +91,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
} else {
hostURL = rawRequest.Headers["Host"]
}
if strings.Contains(hostURL, ":") && strings.Contains(parsedURL.Host, ":") {
parsedURL.Host, _, _ = net.SplitHostPort(parsedURL.Host)
}
if rawRequest.Path == "" {
rawRequest.Path = parsedURL.Path

13
v2/pkg/protocols/http/raw/raw_test.go
View File

@ -6,6 +6,19 @@ import (
"github.com/stretchr/testify/require"
)
func TestParseRawRequestWithPort(t *testing.T) {
request, err := Parse(`GET /gg/phpinfo.php HTTP/1.1
Host: {{Hostname}}:123
Origin: {{BaseURL}}
Connection: close
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Language: en-US,en;q=0.9`, "https://example.com:8080", false)
require.Nil(t, err, "could not parse GET request")
require.Equal(t, "https://{{Hostname}}:123/gg/phpinfo.php", request.FullURL, "Could not parse request url correctly")
require.Equal(t, "/gg/phpinfo.php", request.Path, "Could not parse request path correctly")
}
func TestParseRawRequest(t *testing.T) {
request, err := Parse(`GET /manager/html HTTP/1.1
Host: {{Hostname}}

99
v2/pkg/protocols/http/request.go
View File

@ -26,10 +26,10 @@ import (
const defaultMaxWorkers = 150
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
func (r *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
maxWorkers := e.RaceNumberRequests
maxWorkers := r.RaceNumberRequests
swg := sizedwaitgroup.New(maxWorkers)
var requestErr error
@ -39,10 +39,10 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
if err != nil {
return err
}
for i := 0; i < e.RaceNumberRequests; i++ {
for i := 0; i < r.RaceNumberRequests; i++ {
swg.Add()
go func(httpRequest *generatedRequest) {
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
@ -55,12 +55,12 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
return requestErr
}
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
// executeRaceRequest executes parallel requests for a template
func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
// Workers that keeps enqueuing new requests
maxWorkers := e.Threads
maxWorkers := r.Threads
swg := sizedwaitgroup.New(maxWorkers)
var requestErr error
@ -71,30 +71,30 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
break
}
if err != nil {
e.options.Progress.DecrementRequests(int64(generator.Total()))
r.options.Progress.DecrementRequests(int64(generator.Total()))
return err
}
swg.Add()
go func(httpRequest *generatedRequest) {
defer swg.Done()
e.options.RateLimiter.Take()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
r.options.RateLimiter.Take()
err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
}
mutex.Unlock()
}(request)
e.options.Progress.IncrementRequests()
r.options.Progress.IncrementRequests()
}
swg.Wait()
return requestErr
}
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
// executeRaceRequest executes turbo http request for a URL
func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
// need to extract the target from the url
URL, err := url.Parse(reqURL)
@ -105,11 +105,11 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
pipeOptions := rawhttp.DefaultPipelineOptions
pipeOptions.Host = URL.Host
pipeOptions.MaxConnections = 1
if e.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = e.PipelineConcurrentConnections
if r.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = r.PipelineConcurrentConnections
}
if e.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = e.PipelineRequestsPerConnection
if r.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection
}
pipeclient := rawhttp.NewPipelineClient(pipeOptions)
@ -129,7 +129,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
break
}
if err != nil {
e.options.Progress.DecrementRequests(int64(generator.Total()))
r.options.Progress.DecrementRequests(int64(generator.Total()))
return err
}
request.pipelinedClient = pipeclient
@ -138,14 +138,14 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
go func(httpRequest *generatedRequest) {
defer swg.Done()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
}
mutex.Unlock()
}(request)
e.options.Progress.IncrementRequests()
r.options.Progress.IncrementRequests()
}
swg.Wait()
return requestErr
@ -204,6 +204,8 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
return requestErr
}
const drainReqSize = int64(8 * 1024)
// executeRequest executes the actual generated request and returns error if occured
func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
@ -240,7 +242,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
if parsed, err := url.Parse(formedURL); err == nil {
hostname = parsed.Hostname()
}
request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n")
options := request.original.rawhttpClient.Options
options.AutomaticContentLength = !r.DisableAutoContentLength
options.AutomaticHostHeader = !r.DisableAutoHostname
@ -266,35 +267,46 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
if err != nil {
// rawhttp doesn't supports draining response bodies.
if resp != nil && resp.Body != nil && request.rawRequest == nil {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize)
resp.Body.Close()
}
r.options.Output.Request(r.options.TemplateID, reqURL, "http", err)
r.options.Progress.DecrementRequests(1)
return err
}
gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL)
r.options.Output.Request(r.options.TemplateID, reqURL, "http", err)
duration := time.Since(timeStart)
// Dump response - Step 1 - Decompression not yet handled
var dumpedResponse []byte
if r.options.Options.Debug || r.options.Options.DebugResponse || r.options.Options.JSONRequests {
var dumpErr error
dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
if dumpErr != nil {
return errors.Wrap(dumpErr, "could not dump http response")
}
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
_, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize)
resp.Body.Close()
return errors.Wrap(err, "could not dump http response")
}
data, err := ioutil.ReadAll(resp.Body)
var bodyReader io.Reader
if r.MaxSize != 0 {
bodyReader = io.LimitReader(resp.Body, int64(r.MaxSize))
} else {
bodyReader = resp.Body
}
data, err := ioutil.ReadAll(bodyReader)
if err != nil {
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize)
resp.Body.Close()
return errors.Wrap(err, "could not read http body")
}
resp.Body.Close()
redirectedResponse, err := dumpResponseWithRedirectChain(resp, data)
if err != nil {
return errors.Wrap(err, "could not read http response with redirect chain")
}
// net/http doesn't automatically decompress the response body if an
// encoding has been specified by the user in the request so in case we have to
// manually do it.
@ -305,9 +317,11 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if r.options.Options.Debug || r.options.Options.DebugResponse {
dumpedResponse = data
dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data)
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", string(dumpedResponse))
gologger.Print().Msgf("%s", string(redirectedResponse))
}
// if nuclei-project is enabled store the response if not previously done
@ -327,6 +341,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
}
outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
for k, v := range previous {
outputEvent[k] = v
}
@ -347,12 +362,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
const two = 2
// setCustomHeaders sets the custom headers for generated request
func (e *Request) setCustomHeaders(r *generatedRequest) {
for k, v := range e.customHeaders {
if r.rawRequest != nil {
r.rawRequest.Headers[k] = v
func (r *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range r.customHeaders {
if req.rawRequest != nil {
req.rawRequest.Headers[k] = v
} else {
r.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
req.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
}
}
}

69
v2/pkg/protocols/http/request_generator.go Normal file
View File

@ -0,0 +1,69 @@
package http
import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
// requestGenerator generates requests sequentially based on various
// configurations for a http request template.
//
// If payload values are present, an iterator is created for the payload
// values. Paths and Raw requests are supported as base input, so
// it will automatically select between them based on the template.
type requestGenerator struct {
currentIndex int
request *Request
payloadIterator *generators.Iterator
}
// newGenerator creates a new request generator instance
func (r *Request) newGenerator() *requestGenerator {
generator := &requestGenerator{request: r}
if len(r.Payloads) > 0 {
generator.payloadIterator = r.generator.NewIterator()
}
return generator
}
// nextValue returns the next path or the next raw request depending on user input
// It returns false if all the inputs have been exhausted by the generator instance.
func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) {
// If we have paths, return the next path.
if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) {
if item := r.request.Path[r.currentIndex]; item != "" {
r.currentIndex++
return item, nil, true
}
}
// If we have raw requests, start with the request at current index.
// If we are not at the start, then check if the iterator for payloads
// has finished if there are any.
//
// If the iterator has finished for the current raw request
// then reset it and move on to the next value, otherwise use the last request.
if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) {
if r.payloadIterator != nil {
payload, ok := r.payloadIterator.Value()
if !ok {
r.currentIndex++
r.payloadIterator.Reset()
// No more payloads request for us now.
if len(r.request.Raw) == r.currentIndex {
return "", nil, false
}
if item := r.request.Raw[r.currentIndex]; item != "" {
newPayload, ok := r.payloadIterator.Value()
return item, newPayload, ok
}
return "", nil, false
}
return r.request.Raw[r.currentIndex], payload, true
}
if item := r.request.Raw[r.currentIndex]; item != "" {
r.currentIndex++
return item, nil, true
}
}
return "", nil, false
}

70
v2/pkg/protocols/http/request_generator_test.go Normal file
View File

@ -0,0 +1,70 @@
package http
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/stretchr/testify/require"
)
func TestRequestGeneratorPaths(t *testing.T) {
req := &Request{
Path: []string{"{{BaseURL}}/test", "{{BaseURL}}/test.php"},
}
generator := req.newGenerator()
var payloads []string
for {
raw, _, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, raw)
}
require.Equal(t, req.Path, payloads, "Could not get correct paths")
}
func TestRequestGeneratorClusterBombSingle(t *testing.T) {
var err error
req := &Request{
Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}},
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`},
}
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
generator := req.newGenerator()
var payloads []map[string]interface{}
for {
_, data, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, data)
}
require.Equal(t, 9, len(payloads), "Could not get correct number of payloads")
}
func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {
var err error
req := &Request{
Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}},
attackType: generators.ClusterBomb,
Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},
}
req.generator, err = generators.New(req.Payloads, req.attackType, "")
require.Nil(t, err, "could not create generator")
generator := req.newGenerator()
var payloads []map[string]interface{}
for {
_, data, ok := generator.nextValue()
if !ok {
break
}
payloads = append(payloads, data)
}
require.Equal(t, 18, len(payloads), "Could not get correct number of payloads")
}

49
v2/pkg/protocols/http/utils.go
View File

@ -9,9 +9,58 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/rawhttp"
)
// dumpResponseWithRedirectChain dumps a http response with the
// complete http redirect chain.
//
// It preserves the order in which responses were given to requests
// and returns the data to the user for matching and viewing in that order.
//
// Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923
func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, error) {
redirects := []string{}
respData, err := httputil.DumpResponse(resp, false)
if err != nil {
return nil, err
}
redirectChain := &bytes.Buffer{}
redirectChain.WriteString(tostring.UnsafeToString(respData))
redirectChain.Write(body)
redirects = append(redirects, redirectChain.String())
redirectChain.Reset()
var redirectResp *http.Response
if resp != nil || resp.Request != nil {
redirectResp = resp.Request.Response
}
for redirectResp != nil {
var body []byte
respData, err := httputil.DumpResponse(redirectResp, false)
if err != nil {
break
}
if redirectResp.Body != nil {
body, _ = ioutil.ReadAll(redirectResp.Body)
}
redirectChain.WriteString(tostring.UnsafeToString(respData))
if len(body) > 0 {
redirectChain.WriteString(tostring.UnsafeToString(body))
}
redirects = append(redirects, redirectChain.String())
redirectResp = redirectResp.Request.Response
redirectChain.Reset()
}
for i := len(redirects) - 1; i >= 0; i-- {
redirectChain.WriteString(redirects[i])
}
return redirectChain.Bytes(), nil
}
// headersToString converts http headers to string
func headersToString(headers http.Header) string {
builder := &strings.Builder{}

36
v2/pkg/protocols/network/network_test.go Normal file
View File

@ -0,0 +1,36 @@
package network
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/stretchr/testify/require"
)
func TestNetworkCompileMake(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}", "{{Hostname}}:8082"},
ReadSize: 1024,
Inputs: []*Input{{Data: "test-data"}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
require.Equal(t, 2, len(request.addresses), "could not get correct number of input address")
t.Run("check-host", func(t *testing.T) {
require.Equal(t, "{{Hostname}}", request.addresses[0].key, "could not get correct host")
})
t.Run("check-host-with-port", func(t *testing.T) {
require.Equal(t, "{{Hostname}}", request.addresses[1].key, "could not get correct host with port")
require.Equal(t, "8082", request.addresses[1].value, "could not get correct port for host")
})
}

31
v2/pkg/protocols/network/operators.go
View File

@ -13,7 +13,7 @@ import (
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
partString := matcher.Part
switch partString {
case "body", "all", "":
case "body", "raw", "all", "":
partString = "data"
}
@ -40,14 +40,9 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part, ok := data[extractor.Part]
if !ok {
return nil
}
partString := part.(string)
partString := extractor.Part
switch partString {
case "body", "all":
case "body", "raw", "all", "":
partString = "data"
}
@ -68,14 +63,12 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 4)
data := make(output.InternalEvent, 6)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
if r.options.Options.JSONRequests {
data["request"] = req
}
data["request"] = req
data["data"] = resp
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
@ -112,18 +105,18 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: wrapped.InternalEvent["template-id"].(string),
Info: wrapped.InternalEvent["template-info"].(map[string]string),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "network",
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
IP: wrapped.InternalEvent["ip"].(string),
Timestamp: time.Now(),
IP: types.ToString(wrapped.InternalEvent["ip"]),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)
data.Response = wrapped.InternalEvent["data"].(string)
data.Request = types.ToString(wrapped.InternalEvent["request"])
data.Response = types.ToString(wrapped.InternalEvent["data"])
}
return data
}

198
v2/pkg/protocols/network/operators_test.go Normal file
View File

@ -0,0 +1,198 @@
package network
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}"},
ReadSize: 1024,
Inputs: []*Input{{Data: "test-data\r\n"}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
req := "test-data\r\n"
resp := "resp-data\r\n"
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
require.Len(t, event, 6, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["data"], "could not get correct resp")
}
func TestNetworkOperatorMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}"},
ReadSize: 1024,
Inputs: []*Input{{Data: "test-data\r\n"}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
req := "test-data\r\n"
resp := "resp-data\r\nSTAT \r\n"
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Words: []string{"STAT "},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
})
t.Run("negative", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Negative: true,
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
})
t.Run("invalid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "raw",
Type: "word",
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
})
}
func TestNetworkOperatorExtract(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}"},
ReadSize: 1024,
Inputs: []*Input{{Data: "test-data\r\n"}},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
req := "test-data\r\n"
resp := "resp-data\r\nSTAT \r\n1.1.1.1\r\n"
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor valid response")
require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
})
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
KVal: []string{"request"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile kval extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor kval valid response")
require.Equal(t, map[string]struct{}{req: {}}, data, "could not extract correct kval data")
})
}
func TestNetworkMakeResult(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}"},
ReadSize: 1024,
Inputs: []*Input{{Data: "test-data\r\n"}},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"STAT "},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
req := "test-data\r\n"
resp := "resp-data\rSTAT \r\n1.1.1.1\n"
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
event["ip"] = "192.168.1.1"
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}

3
v2/pkg/protocols/network/request.go
View File

@ -27,8 +27,7 @@ func (r *Request) ExecuteWithResults(input string, metadata, previous output.Int
}
for _, kv := range r.addresses {
replacer := replacer.New(map[string]interface{}{"Hostname": address})
actualAddress := replacer.Replace(kv.key)
actualAddress := replacer.Replace(kv.key, map[string]interface{}{"Hostname": address})
if kv.value != "" {
if strings.Contains(address, ":") {
actualAddress, _, _ = net.SplitHostPort(actualAddress)

93
v2/pkg/protocols/network/request_test.go Normal file
View File

@ -0,0 +1,93 @@
package network
import (
"encoding/hex"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestNetworkExecuteWithResults(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-network"
request := &Request{
ID: templateID,
Address: []string{"{{Hostname}}:80"},
ReadSize: 2048,
Inputs: []*Input{{Data: "GET / HTTP/1.1\r\n\r\n"}},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"400 - Bad Request"},
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Regex: []string{"<h1>.*</h1>"},
}},
},
}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile network request")
var finalEvent *output.InternalWrappedEvent
t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
t.Run("invalid-port-override", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults("example.com:11211", metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
finalEvent = nil
request.Inputs[0].Type = "hex"
request.Inputs[0].Data = hex.EncodeToString([]byte("GET / HTTP/1.1\r\n\r\n"))
t.Run("hex-to-string", func(t *testing.T) {
metadata := make(output.InternalEvent)
previous := make(output.InternalEvent)
err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event
})
require.Nil(t, err, "could not execute network request")
})
require.NotNil(t, finalEvent, "could not get event output from request")
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
require.Equal(t, "<h1>400 - Bad Request</h1>", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}

106
v2/pkg/protocols/offlinehttp/find.go Normal file
View File

@ -0,0 +1,106 @@
package offlinehttp
import (
"os"
"path"
"path/filepath"
"strings"
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
)
// getInputPaths parses the specified input paths and returns a compiled
// list of finished absolute paths to the files evaluating any allowlist, denylist,
// glob, file or folders, etc.
func (r *Request) getInputPaths(target string, callback func(string)) error {
processed := make(map[string]struct{})
// Template input includes a wildcard
if strings.Contains(target, "*") {
err := r.findGlobPathMatches(target, processed, callback)
if err != nil {
return errors.Wrap(err, "could not find glob matches")
}
return nil
}
// Template input is either a file or a directory
file, err := r.findFileMatches(target, processed, callback)
if err != nil {
return errors.Wrap(err, "could not find file")
}
if file {
return nil
}
// Recursively walk down the Templates directory and run all
// the template file checks
err = r.findDirectoryMatches(target, processed, callback)
if err != nil {
return errors.Wrap(err, "could not find directory matches")
}
return nil
}
// findGlobPathMatches returns the matched files from a glob path
func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
matches, err := filepath.Glob(absPath)
if err != nil {
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
for _, match := range matches {
if path.Ext(match) != ".txt" {
continue // only process .txt files
}
if _, ok := processed[match]; !ok {
processed[match] = struct{}{}
callback(match)
}
}
return nil
}
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) {
info, err := os.Stat(absPath)
if err != nil {
return false, err
}
if !info.Mode().IsRegular() {
return false, nil
}
if path.Ext(absPath) != ".txt" {
return false, nil // only process .txt files
}
if _, ok := processed[absPath]; !ok {
processed[absPath] = struct{}{}
callback(absPath)
}
return true, nil
}
// findDirectoryMatches finds matches for templates from a directory
func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
err := godirwalk.Walk(absPath, &godirwalk.Options{
Unsorted: true,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Callback: func(p string, d *godirwalk.Dirent) error {
if d.IsDir() {
return nil
}
if path.Ext(p) != ".txt" {
return nil // only process .txt files
}
if _, ok := processed[p]; !ok {
callback(p)
processed[p] = struct{}{}
}
return nil
},
})
return err
}

59
v2/pkg/protocols/offlinehttp/find_test.go Normal file
View File

@ -0,0 +1,59 @@
package offlinehttp
import (
"io/ioutil"
"os"
"path"
"testing"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/stretchr/testify/require"
)
func TestFindResponses(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-offline"
request := &Request{}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
executerOpts.Operators = []*operators.Operators{&operators.Operators{}}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
tempDir, err := ioutil.TempDir("", "test-*")
require.Nil(t, err, "could not create temporary directory")
defer os.RemoveAll(tempDir)
files := map[string]string{
"test.go": "TEST",
"config.txt": "TEST",
"final.txt": "TEST",
"image_ignored.png": "TEST",
"test.txt": "TEST",
}
for k, v := range files {
err = ioutil.WriteFile(path.Join(tempDir, k), []byte(v), 0777)
require.Nil(t, err, "could not write temporary file")
}
expected := []string{"config.txt", "final.txt", "test.txt"}
got := []string{}
err = request.getInputPaths(tempDir+"/*", func(item string) {
base := path.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for glob")
require.ElementsMatch(t, expected, got, "could not get correct file matches for glob")
got = []string{}
err = request.getInputPaths(tempDir, func(item string) {
base := path.Base(item)
got = append(got, base)
})
require.Nil(t, err, "could not get input paths for directory")
require.ElementsMatch(t, expected, got, "could not get correct file matches for directory")
}

35
v2/pkg/protocols/offlinehttp/offlinehttp.go Normal file
View File

@ -0,0 +1,35 @@
package offlinehttp
import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
// Request is a offline http response processing request
type Request struct {
options *protocols.ExecuterOptions
compiledOperators []*operators.Operators
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return ""
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
for _, operator := range options.Operators {
if err := operator.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.compiledOperators = append(r.compiledOperators, operator)
}
r.options = options
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
return 1
}

151
v2/pkg/protocols/offlinehttp/operators.go Normal file
View File

@ -0,0 +1,151 @@
package offlinehttp
import (
"net/http"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
item, ok := getMatchPart(matcher.Part, data)
if !ok {
return false
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
if !ok {
return false
}
return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(item)))
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(item))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(item))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(item))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
}
return false
}
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
item, ok := getMatchPart(extractor.Part, data)
if !ok {
return nil
}
switch extractor.GetType() {
case extractors.RegexExtractor:
return extractor.ExtractRegex(item)
case extractors.KValExtractor:
return extractor.ExtractKval(data)
}
return nil
}
// getMatchPart returns the match part honoring "all" matchers + others.
func getMatchPart(part string, data output.InternalEvent) (string, bool) {
if part == "header" {
part = "all_headers"
}
var itemStr string
if part == "all" {
builder := &strings.Builder{}
builder.WriteString(types.ToString(data["body"]))
builder.WriteString(types.ToString(data["all_headers"]))
itemStr = builder.String()
} else {
item, ok := data[part]
if !ok {
return "", false
}
itemStr = types.ToString(item)
}
return itemStr, true
}
// responseToDSLMap converts a HTTP response to a map for use in DSL matching
func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}
data["host"] = host
data["matched"] = matched
data["request"] = rawReq
data["response"] = rawResp
data["content_length"] = resp.ContentLength
data["status_code"] = resp.StatusCode
data["body"] = body
for _, cookie := range resp.Cookies() {
data[strings.ToLower(cookie.Name)] = cookie.Value
}
for k, v := range resp.Header {
k = strings.ToLower(strings.TrimSpace(k))
data[k] = strings.Join(v, " ")
}
data["all_headers"] = headers
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
return data
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Host: types.ToString(wrapped.InternalEvent["host"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
IP: types.ToString(wrapped.InternalEvent["ip"]),
}
if r.options.Options.JSONRequests {
data.Request = types.ToString(wrapped.InternalEvent["request"])
data.Response = types.ToString(wrapped.InternalEvent["raw"])
}
return data
}

290
v2/pkg/protocols/offlinehttp/operators_test.go Normal file
View File

@ -0,0 +1,290 @@
package offlinehttp
import (
"net/http"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestResponseToDSLMap(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
executerOpts.Operators = []*operators.Operators{&operators.Operators{}}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
}
func TestHTTPOperatorMatch(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
executerOpts.Operators = []*operators.Operators{&operators.Operators{}}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
t.Run("valid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Words: []string{"1.1.1.1"},
}
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
})
t.Run("negative", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Negative: true,
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
})
t.Run("invalid", func(t *testing.T) {
matcher := &matchers.Matcher{
Part: "body",
Type: "word",
Words: []string{"random"},
}
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
})
}
func TestHTTPOperatorExtract(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
executerOpts.Operators = []*operators.Operators{&operators.Operators{}}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test-Header", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header")
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor valid response")
require.Equal(t, map[string]struct{}{"1.1.1.1": {}}, data, "could not extract correct data")
})
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
KVal: []string{"test-header"},
}
err = extractor.CompileExtractors()
require.Nil(t, err, "could not compile kval extractor")
data := request.Extract(event, extractor)
require.Greater(t, len(data), 0, "could not extractor kval valid response")
require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data")
})
}
func TestHTTPMakeResult(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
templateID := "testing-http"
request := &Request{}
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: map[string]interface{}{"severity": "low", "name": "test"},
})
executerOpts.Operators = []*operators.Operators{&operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "body",
Type: "word",
Words: []string{"1.1.1.1"},
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
}}
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := &http.Response{}
resp.Header = make(http.Header)
resp.Header.Set("Test", "Test-Response")
host := "http://example.com/test/"
matched := "http://example.com/test/?test=1"
event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})
require.Len(t, event, 12, "could not get correct number of items in dsl map")
require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp")
require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header")
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
for _, operator := range request.compiledOperators {
result, ok := operator.Execute(event, request.Match, request.Extract)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
}
const exampleRawRequest = `GET / HTTP/1.1
Host: example.com
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.9,hi;q=0.8
If-None-Match: "3147526947+gzip"
If-Modified-Since: Thu, 17 Oct 2019 07:18:26 GMT
Connection: close
`
const exampleRawResponse = exampleResponseHeader + exampleResponseBody
const exampleResponseHeader = `
HTTP/1.1 200 OK
Accept-Ranges: bytes
Age: 493322
Cache-Control: max-age=604800
Content-Type: text/html; charset=UTF-8
Date: Thu, 04 Feb 2021 12:15:51 GMT
Etag: "3147526947+ident"
Expires: Thu, 11 Feb 2021 12:15:51 GMT
Last-Modified: Thu, 17 Oct 2019 07:18:26 GMT
Server: ECS (nyb/1D1C)
Vary: Accept-Encoding
X-Cache: HIT
Content-Length: 1256
Connection: close
`
const exampleResponseBody = `
<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8" />
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style type="text/css">
body {
background-color: #f0f0f2;
margin: 0;
padding: 0;
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
}
div {
width: 600px;
margin: 5em auto;
padding: 2em;
background-color: #fdfdff;
border-radius: 0.5em;
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
}
a:link, a:visited {
color: #38488f;
text-decoration: none;
}
@media (max-width: 700px) {
div {
margin: 0 auto;
width: auto;
}
}
</style>
</head>
<a>1.1.1.1</a>
<body>
<div>
<h1>Example Domain</h1>
<p>This domain is for use in illustrative examples in documents. You may use this
domain in literature without prior coordination or asking for permission.</p>
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
`

18
v2/pkg/protocols/offlinehttp/read_response.go Normal file
View File

@ -0,0 +1,18 @@
package offlinehttp
import (
"bufio"
"net/http"
"strings"
)
// readResponseFromString reads a raw http response from a string.
func readResponseFromString(data string) (*http.Response, error) {
var final string
if strings.HasPrefix(data, "HTTP/") {
final = data
} else {
final = data[strings.LastIndex(data, "HTTP/"):] // choose last http/ in case of it being later.
}
return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil)
}

85
v2/pkg/protocols/offlinehttp/read_response_test.go Normal file
View File

@ -0,0 +1,85 @@
package offlinehttp
import (
"io/ioutil"
"testing"
"github.com/stretchr/testify/require"
)
func TestReadResponseFromString(t *testing.T) {
expectedBody := `<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</body>
</html>`
t.Run("response", func(t *testing.T) {
data := `HTTP/1.1 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</body>
</html>`
resp, err := readResponseFromString(data)
require.Nil(t, err, "could not read response from string")
respData, err := ioutil.ReadAll(resp.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
t.Run("request-response", func(t *testing.T) {
data := `GET http://public-firing-range.appspot.com/ HTTP/1.1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36
HTTP/1.1 200 OK
Age: 0
Cache-Control: public, max-age=600
Content-Type: text/html
Server: Google Frontend
<!DOCTYPE html>
<html>
<head>
<title>Firing Range</title>
</head>
<body>
<h1>Version 0.48</h1>
<h1>What is the Firing Range?</h1>
<p>
</body>
</body>
</html>`
resp, err := readResponseFromString(data)
require.Nil(t, err, "could not read response from string")
respData, err := ioutil.ReadAll(resp.Body)
require.Nil(t, err, "could not read response body")
require.Equal(t, expectedBody, string(respData), "could not get correct parsed body")
require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers")
})
}

128
v2/pkg/protocols/offlinehttp/request.go Normal file
View File

@ -0,0 +1,128 @@
package offlinehttp
import (
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/remeh/sizedwaitgroup"
)
var _ protocols.Request = &Request{}
const maxSize = 5 * 1024 * 1024
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(r.options.Options.RateLimit)
err := r.getInputPaths(input, func(data string) {
wg.Add()
go func(data string) {
defer wg.Done()
file, err := os.Open(data)
if err != nil {
gologger.Error().Msgf("Could not open file path %s: %s\n", data, err)
return
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err)
return
}
if stat.Size() >= int64(maxSize) {
gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data)
return
}
buffer, err := ioutil.ReadAll(file)
if err != nil {
gologger.Error().Msgf("Could not read file path %s: %s\n", data, err)
return
}
dataStr := tostring.UnsafeToString(buffer)
resp, err := readResponseFromString(dataStr)
if err != nil {
gologger.Error().Msgf("Could not read raw response %s: %s\n", data, err)
return
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped offline-http request for %s", r.options.TemplateID, data)
gologger.Print().Msgf("%s", dataStr)
}
gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", r.options.TemplateID, data)
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
gologger.Error().Msgf("Could not dump raw http response %s: %s\n", data, err)
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
gologger.Error().Msgf("Could not read raw http response body %s: %s\n", data, err)
return
}
outputEvent := r.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
outputEvent["ip"] = ""
for k, v := range previous {
outputEvent[k] = v
}
for _, operator := range r.compiledOperators {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
var ok bool
event.OperatorsResult, ok = operator.Execute(outputEvent, r.Match, r.Extract)
if ok && event.OperatorsResult != nil {
event.Results = r.MakeResultEvent(event)
}
callback(event)
}
}(data)
})
wg.Wait()
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "file", err)
r.options.Progress.DecrementRequests(1)
return errors.Wrap(err, "could not send file request")
}
r.options.Progress.IncrementRequests()
return nil
}
// headersToString converts http headers to string
func headersToString(headers http.Header) string {
builder := &strings.Builder{}
for header, values := range headers {
builder.WriteString(header)
builder.WriteString(": ")
for i, value := range values {
builder.WriteString(value)
if i != len(values)-1 {
builder.WriteRune('\n')
builder.WriteString(header)
builder.WriteString(": ")
}
}
builder.WriteRune('\n')
}
return builder.String()
}

5
v2/pkg/protocols/protocols.go
View File

@ -3,6 +3,7 @@ package protocols
import (
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/catalogue"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -31,7 +32,7 @@ type ExecuterOptions struct {
// TemplatePath is the path of the template for the request
TemplatePath string
// TemplateInfo contains information block of the template request
TemplateInfo map[string]string
TemplateInfo map[string]interface{}
// Output is a writer interface for writing output events from executer.
Output output.Writer
// Options contains configuration options for the executer.
@ -46,6 +47,8 @@ type ExecuterOptions struct {
Catalogue *catalogue.Catalogue
// ProjectFile is the project file for nuclei
ProjectFile *projectfile.ProjectFile
Operators []*operators.Operators // only used by offlinehttp module
}
// Request is an interface implemented any protocol based request generator.

102
v2/pkg/templates/compile.go
View File

@ -3,16 +3,20 @@ package templates
import (
"fmt"
"os"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"gopkg.in/yaml.v2"
)
// Parse parses a yaml request template file
func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, error) {
func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error) {
template := &Template{}
f, err := os.Open(filePath)
@ -26,6 +30,22 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro
}
defer f.Close()
if _, ok := template.Info["name"]; !ok {
return nil, errors.New("no template name field provided")
}
if _, ok := template.Info["author"]; !ok {
return nil, errors.New("no template author field provided")
}
if len(options.Options.Tags) > 0 {
templateTags, ok := template.Info["tags"]
if !ok {
return nil, errors.New("no tags found for template")
}
if err := matchTemplateWithTags(types.ToString(templateTags), options.Options); err != nil {
return nil, err
}
}
// Setting up variables regarding template metadata
options.TemplateID = template.ID
options.TemplateInfo = template.Info
@ -39,7 +59,7 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro
// Compile the workflow request
if len(template.Workflows) > 0 {
compiled := &template.Workflow
if err := template.compileWorkflow(options, compiled); err != nil {
if err := template.compileWorkflow(&options, compiled); err != nil {
return nil, errors.Wrap(err, "could not compile workflow")
}
template.CompiledWorkflow = compiled
@ -47,29 +67,39 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro
// Compile the requests found
requests := []protocols.Request{}
if len(template.RequestsDNS) > 0 {
if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP {
for _, req := range template.RequestsDNS {
requests = append(requests, req)
}
template.Executer = executer.NewExecuter(requests, options)
template.Executer = executer.NewExecuter(requests, &options)
}
if len(template.RequestsHTTP) > 0 {
for _, req := range template.RequestsHTTP {
requests = append(requests, req)
if options.Options.OfflineHTTP {
operators := []*operators.Operators{}
for _, req := range template.RequestsHTTP {
operators = append(operators, &req.Operators)
}
options.Operators = operators
template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
} else {
for _, req := range template.RequestsHTTP {
requests = append(requests, req)
}
template.Executer = executer.NewExecuter(requests, &options)
}
template.Executer = executer.NewExecuter(requests, options)
}
if len(template.RequestsFile) > 0 {
if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP {
for _, req := range template.RequestsFile {
requests = append(requests, req)
}
template.Executer = executer.NewExecuter(requests, options)
template.Executer = executer.NewExecuter(requests, &options)
}
if len(template.RequestsNetwork) > 0 {
if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP {
for _, req := range template.RequestsNetwork {
requests = append(requests, req)
}
template.Executer = executer.NewExecuter(requests, options)
template.Executer = executer.NewExecuter(requests, &options)
}
if template.Executer != nil {
err := template.Executer.Compile()
@ -78,6 +108,9 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro
}
template.TotalRequests += template.Executer.Requests()
}
if template.Executer == nil {
return nil, errors.New("cannot create template executer")
}
return template, nil
}
@ -118,7 +151,7 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
return errors.Wrap(err, "could not get workflow template")
}
for _, path := range paths {
opts := &protocols.ExecuterOptions{
opts := protocols.ExecuterOptions{
Output: options.Output,
Options: options.Options,
Progress: options.Progress,
@ -140,3 +173,48 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
}
return nil
}
// matchTemplateWithTags matches if the template matches a tag
func matchTemplateWithTags(tags string, options *types.Options) error {
actualTags := strings.Split(tags, ",")
matched := false
mainLoop:
for _, t := range options.Tags {
commaTags := strings.Split(t, ",")
for _, tag := range commaTags {
tag = strings.TrimSpace(tag)
key, value := getKeyValue(tag)
for _, templTag := range actualTags {
templTag = strings.TrimSpace(templTag)
tKey, tValue := getKeyValue(templTag)
if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
matched = true
break mainLoop
}
}
}
}
if !matched {
return errors.New("could not match template tags with input")
}
return nil
}
// getKeyValue returns key value pair for a data string
func getKeyValue(data string) (string, string) {
var key, value string
if strings.Contains(data, ":") {
parts := strings.SplitN(data, ":", 2)
if len(parts) == 2 {
key, value = parts[0], parts[1]
}
}
if value == "" {
value = data
}
return key, value
}

32
v2/pkg/templates/compile_test.go Normal file
View File

@ -0,0 +1,32 @@
package templates
import (
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/stretchr/testify/require"
)
func TestMatchTemplateWithTags(t *testing.T) {
err := matchTemplateWithTags("php,linux,symfony", &types.Options{Tags: []string{"php"}})
require.Nil(t, err, "could not get php tag from input slice")
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", &types.Options{Tags: []string{"cms:symfony"}})
require.Nil(t, err, "could not get php tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,symfony", &types.Options{Tags: []string{"cms:symfony"}})
require.NotNil(t, err, "could get key value tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,cms:jira", &types.Options{Tags: []string{"cms:symfony"}})
require.NotNil(t, err, "could get key value tag from input key value")
t.Run("space", func(t *testing.T) {
err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", &types.Options{Tags: []string{"cms:symfony"}})
require.Nil(t, err, "could get key value tag from input key value with space")
})
t.Run("comma-tags", func(t *testing.T) {
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", &types.Options{Tags: []string{"test,cms:symfony"}})
require.Nil(t, err, "could get key value tag from input key value with comma")
})
}

4
v2/pkg/templates/templates.go
View File

@ -14,7 +14,7 @@ type Template struct {
// ID is the unique id for the template
ID string `yaml:"id"`
// Info contains information about the template
Info map[string]string `yaml:"info"`
Info map[string]interface{} `yaml:"info"`
// RequestsHTTP contains the http request to make in the template
RequestsHTTP []*http.Request `yaml:"requests,omitempty"`
// RequestsDNS contains the dns request to make in the template
@ -23,7 +23,7 @@ type Template struct {
RequestsFile []*file.Request `yaml:"file,omitempty"`
// RequestsNetwork contains the network request to make in the template
RequestsNetwork []*network.Request `yaml:"network,omitempty"`
// Workflows is a yaml based workflow declaration code.
workflows.Workflow `yaml:",inline"`
CompiledWorkflow *workflows.Workflow

2
v2/pkg/types/interfaces.go
View File

@ -11,6 +11,8 @@ import (
// ToString converts an interface to string in a quick way
func ToString(data interface{}) string {
switch s := data.(type) {
case nil:
return ""
case string:
return s
case bool:

8
v2/pkg/types/types.go
View File

@ -84,4 +84,12 @@ type Options struct {
ReportingDB string
// ReportingConfig is the config file for nuclei reporting module
ReportingConfig string
// Tags contains a list of tags to execute templates for. Multiple paths
// can be specified with -l flag and -tags can be used in combination with
// the -l flag.
Tags goflags.StringSlice
// OfflineHTTP is a flag that specific offline processing of http response
// using same matchers/extractors from http protocol without the need
// to send a new request, reading responses from a file.
OfflineHTTP bool
}

6
v2/pkg/workflows/execute.go
View File

@ -34,7 +34,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 {
mainErr = err
} else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err)
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
}
continue
}
@ -76,7 +76,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 {
mainErr = err
} else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err)
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
}
continue
}
@ -84,7 +84,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
if len(template.Executers) == 1 {
mainErr = executionErr
} else {
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", executionErr)
gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, executionErr)
}
}
}

132
v2/pkg/workflows/execute_test.go
View File

@ -14,11 +14,10 @@ func TestWorkflowsSimple(t *testing.T) {
progress, _ := progress.NewProgress(false, false, 0)
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true}}},
},
options: &protocols.ExecuterOptions{
Progress: progress,
}}
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progress}},
}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")
@ -30,14 +29,17 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}}}},
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}}},
},
options: &protocols.ExecuterOptions{Progress: progress}}
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}},
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")
@ -52,16 +54,16 @@ func TestWorkflowsSubtemplates(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}}},
Subtemplates: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}}},
},
options: &protocols.ExecuterOptions{Progress: progress}}
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")
@ -76,16 +78,16 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: false, executeHook: func(input string) {
firstInput = input
}}},
Subtemplates: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}}},
},
options: &protocols.ExecuterOptions{Progress: progress}}
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: false, executeHook: func(input string) {
firstInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")
@ -100,24 +102,21 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Extracts: map[string][]string{},
}},
}}},
Matchers: []*Matcher{
{Name: "tomcat", Subtemplates: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}}},
},
},
},
options: &protocols.ExecuterOptions{Progress: progress}}
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Extracts: map[string][]string{},
}},
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}}}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")
@ -132,24 +131,21 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Extracts: map[string][]string{},
}},
}}},
Matchers: []*Matcher{
{Name: "apache", Subtemplates: []*WorkflowTemplate{
{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}}},
{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Extracts: map[string][]string{},
}},
},
},
},
options: &protocols.ExecuterOptions{Progress: progress}}
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}, Options: &protocols.ExecuterOptions{Progress: progress}},
}}}}}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
require.Nil(t, err, "could not run workflow")