From deaa86f5cc3beb500d7bfcd2aad8078c07bac4ec Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 16 Sep 2021 21:29:58 +0530 Subject: [PATCH 001/196] Added interactsh based template to integration test --- integration_tests/http/interactsh.yaml | 19 +++++++++++++++++ v2/cmd/integration-test/http.go | 29 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 integration_tests/http/interactsh.yaml diff --git a/integration_tests/http/interactsh.yaml b/integration_tests/http/interactsh.yaml new file mode 100644 index 000000000..28d9c5606 --- /dev/null +++ b/integration_tests/http/interactsh.yaml @@ -0,0 +1,19 @@ +id: interactsh-integration-test + +info: + name: Interactsh Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 41f177d28..37f1fadf0 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -32,6 +32,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, } func httpDebugRequestDump(r *http.Request) { @@ -42,6 +43,34 @@ func httpDebugRequestDump(r *http.Request) { } } +type httpInteractshRequest struct{} + +// Executes executes a test case and returns an error if occurred +func (h *httpInteractshRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + httpDebugRequestDump(r) + + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpGetHeaders struct{} // Executes executes a test case and returns an error if occurred From 5c31b75ac79af453615aaf9cc3279d7496592aca Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 22 Sep 2021 22:41:07 +0530 Subject: [PATCH 002/196] Added ssl protocol to nuclei --- v2/pkg/operators/common/dsl/dsl.go | 116 +++++++++++++ v2/pkg/protocols/others/others.go | 1 + v2/pkg/protocols/others/ssl/ssl.go | 208 ++++++++++++++++++++++++ v2/pkg/protocols/others/ssl/ssl_test.go | 28 ++++ v2/pkg/templates/compile.go | 64 +++++--- v2/pkg/templates/templates.go | 4 + 6 files changed, 397 insertions(+), 24 deletions(-) create mode 100644 v2/pkg/protocols/others/others.go create mode 100644 v2/pkg/protocols/others/ssl/ssl.go create mode 100644 v2/pkg/protocols/others/ssl/ssl_test.go diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 056f4ace7..0b99505e3 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -31,21 +31,38 @@ const ( withMaxRandArgsSize = withCutSetArgsSize ) +var ErrDSLArguments = errors.New("invalid arguments provided to dsl") + var functions = map[string]govaluate.ExpressionFunction{ "len": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } length := len(types.ToString(args[0])) return float64(length), nil }, "toupper": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.ToUpper(types.ToString(args[0])), nil }, "tolower": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.ToLower(types.ToString(args[0])), nil }, "replace": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil }, "replace_regex": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } compiled, err := regexp.Compile(types.ToString(args[1])) if err != nil { return nil, err @@ -53,66 +70,120 @@ var functions = map[string]govaluate.ExpressionFunction{ return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil }, "trim": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimleft": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimright": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimspace": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return strings.TrimSpace(types.ToString(args[0])), nil }, "trimprefix": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimsuffix": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil }, "reverse": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return reverseString(types.ToString(args[0])), nil }, // encoding "base64": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return sEnc, nil }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" "base64_py": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return deserialization.InsertInto(sEnc, 76, '\n'), nil }, "base64_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return base64.StdEncoding.DecodeString(types.ToString(args[0])) }, "url_encode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return url.QueryEscape(types.ToString(args[0])), nil }, "url_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return url.QueryUnescape(types.ToString(args[0])) }, "hex_encode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return hex.EncodeToString([]byte(types.ToString(args[0]))), nil }, "hex_decode": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } hx, _ := hex.DecodeString(types.ToString(args[0])) return string(hx), nil }, "html_escape": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return html.EscapeString(types.ToString(args[0])), nil }, "html_unescape": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return html.UnescapeString(types.ToString(args[0])), nil }, // hashing "md5": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } hash := md5.Sum([]byte(types.ToString(args[0]))) return hex.EncodeToString(hash[:]), nil }, "sha256": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } h := sha256.New() if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err @@ -120,6 +191,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return hex.EncodeToString(h.Sum(nil)), nil }, "sha1": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } h := sha1.New() if _, err := h.Write([]byte(types.ToString(args[0]))); err != nil { return nil, err @@ -127,13 +201,22 @@ var functions = map[string]govaluate.ExpressionFunction{ return hex.EncodeToString(h.Sum(nil)), nil }, "mmh3": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil }, // search "contains": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil }, "regex": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } compiled, err := regexp.Compile(types.ToString(args[0])) if err != nil { return nil, err @@ -142,6 +225,9 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // random generators "rand_char": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } chars := letters + numbers bad := "" if len(args) >= 1 { @@ -154,6 +240,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return chars[rand.Intn(len(chars))], nil }, "rand_base": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } l := 0 bad := "" base := letters + numbers @@ -171,6 +260,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(base, l), nil }, "rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := letters + numbers @@ -185,6 +277,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_text_alpha": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := letters @@ -199,6 +294,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_text_numeric": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } l := 0 bad := "" chars := numbers @@ -213,6 +311,9 @@ var functions = map[string]govaluate.ExpressionFunction{ return randSeq(chars, l), nil }, "rand_int": func(args ...interface{}) (interface{}, error) { + if len(args) != 2 { + return nil, ErrDSLArguments + } min := 0 max := math.MaxInt32 @@ -226,12 +327,18 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } seconds := args[0].(float64) time.Sleep(time.Duration(seconds) * time.Second) return true, nil }, // deserialization Functions "generate_java_gadget": func(args ...interface{}) (interface{}, error) { + if len(args) != 3 { + return nil, ErrDSLArguments + } gadget := args[0].(string) cmd := args[1].(string) @@ -247,6 +354,15 @@ var functions = map[string]govaluate.ExpressionFunction{ gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) return true, nil }, + // is_before_now compares a timestamp and returns true if the first + // passed argument is a time.Time that has already passed. + "is_time_before_now": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } + isBefore := time.Since(args[0].(time.Time)) > 0 + return isBefore, nil + }, } // HelperFunctions returns the dsl helper functions diff --git a/v2/pkg/protocols/others/others.go b/v2/pkg/protocols/others/others.go new file mode 100644 index 000000000..24b5bd240 --- /dev/null +++ b/v2/pkg/protocols/others/others.go @@ -0,0 +1 @@ +package others diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go new file mode 100644 index 000000000..09fb23011 --- /dev/null +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -0,0 +1,208 @@ +package ssl + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the SSL protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// Match performs matching operation for a matcher on model and returns true or false. +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partItem, ok := data[matcher.Part] + if !ok { + return false + } + item := types.ToString(partItem) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(item))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(item)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(item)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(item)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + address, err := getAddress(input) + if err != nil { + return nil + } + fmt.Printf("address: %v\n", address) + hostname, _, _ := net.SplitHostPort(input) + + config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} + conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + + connTLS, ok := conn.(*tls.Conn) + if !ok { + return nil + } + if len(connTLS.ConnectionState().PeerCertificates) == 0 { + return nil + } + data := make(map[string]interface{}) + cert := connTLS.ConnectionState().PeerCertificates[0] + data["host"] = input + data["not_after"] = cert.NotAfter + data["ip"] = r.dialer.GetDialedIP(hostname) + + event := &output.InternalWrappedEvent{InternalEvent: data} + if r.CompiledOperators != nil { + var ok bool + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, r.Match, r.Extract) + if ok && event.OperatorsResult != nil { + event.Results = r.makeResultEvent(event) + } + callback(event) + } + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if strings.Contains(toTest, "://") { + parsed, err := url.Parse(toTest) + if err != nil { + return "", err + } + _, port, _ := net.SplitHostPort(parsed.Host) + + if parsed.Scheme == "https" && port == "" { + toTest = net.JoinHostPort(parsed.Host, "443") + } else { + toTest = parsed.Host + } + } + return toTest, nil +} + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + Info: r.options.TemplateInfo, + Type: "ssl", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + } + return data +} diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/others/ssl/ssl_test.go new file mode 100644 index 000000000..a043518ab --- /dev/null +++ b/v2/pkg/protocols/others/ssl/ssl_test.go @@ -0,0 +1,28 @@ +package ssl + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestSSLProtocol(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-ssl" + request := &Request{} + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile ssl request") + + err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) + require.Nil(t, err, "could not run ssl request") +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index d7f545bc3..72d880625 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -70,7 +70,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestsSSL) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -85,12 +85,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // Compile the requests found requests := []protocols.Request{} - if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } + if len(template.RequestsHTTP) > 0 { if options.Options.OfflineHTTP { operatorsList := []*operators.Operators{} @@ -115,24 +110,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.Executer = executer.NewExecuter(requests, &options) } } - if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) + if !options.Options.OfflineHTTP { + makeRequestsForTemplate(template, options) } + if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") @@ -147,3 +128,38 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute parsedTemplatesCache.Store(filePath, template, err) return template, nil } + +func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) { + requests := []protocols.Request{} + + if len(template.RequestsDNS) > 0 { + for _, req := range template.RequestsDNS { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsFile) > 0 { + for _, req := range template.RequestsFile { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsNetwork) > 0 { + for _, req := range template.RequestsNetwork { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsHeadless) > 0 && options.Options.Headless { + for _, req := range template.RequestsHeadless { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } + if len(template.RequestsSSL) > 0 { + for _, req := range template.RequestsSSL { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491ac..6a19d2e52 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -56,6 +57,9 @@ type Template struct { // description: | // Headless contains the headless request to make in the template. RequestsHeadless []*headless.Request `yaml:"headless,omitempty" json:"headless,omitempty" jsonschema:"title=headless requests to make,description=Headless requests to make for the template"` + // description: | + // SSL contains the SSL request to make in the template. + RequestsSSL []*ssl.Request `yaml:"ssl,omitempty" json:"ssl,omitempty" jsonschema:"title=ssl requests to make,description=SSL requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. From f6e9acf06f8295a9ecfd215574088cf8da481fb5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 23 Sep 2021 18:30:44 +0530 Subject: [PATCH 003/196] More work on other protocol additions --- v2/go.mod | 2 +- v2/go.sum | 2 + v2/pkg/protocols/others/ssl/ssl.go | 77 +----------------------- v2/pkg/protocols/others/utils/utils.go | 83 ++++++++++++++++++++++++++ v2/pkg/protocols/others/wss/wss.go | 1 + v2/pkg/protocols/protocols.go | 6 -- v2/pkg/templates/compile.go | 15 ++++- 7 files changed, 104 insertions(+), 82 deletions(-) create mode 100644 v2/pkg/protocols/others/utils/utils.go create mode 100644 v2/pkg/protocols/others/wss/wss.go diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..3a3e5eb3d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -30,7 +30,7 @@ require ( github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..82134aed3 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -341,6 +341,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb h1:h+HvVw51KUvcO4Tww1QCd95D6MWV/6wpXuSbmFpPQSI= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210923125921-675fa1873feb/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index 09fb23011..6a33f27e4 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -3,7 +3,6 @@ package ssl import ( "context" "crypto/tls" - "fmt" "net" "net/url" "strings" @@ -12,11 +11,10 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -61,53 +59,12 @@ func (r *Request) GetID() string { return "" } -// Match performs matching operation for a matcher on model and returns true or false. -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { - partItem, ok := data[matcher.Part] - if !ok { - return false - } - item := types.ToString(partItem) - - switch matcher.GetType() { - case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) - case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) - case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) - case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) - case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) - } - return false -} - -// Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := data[extractor.Part] - if !ok { - return nil - } - itemStr := types.ToString(item) - - switch extractor.GetType() { - case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) - case extractors.KValExtractor: - return extractor.ExtractKval(data) - } - return nil -} - // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { return nil } - fmt.Printf("address: %v\n", address) hostname, _, _ := net.SplitHostPort(input) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} @@ -136,9 +93,9 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu event := &output.InternalWrappedEvent{InternalEvent: data} if r.CompiledOperators != nil { var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(data, r.Match, r.Extract) + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) if ok && event.OperatorsResult != nil { - event.Results = r.makeResultEvent(event) + event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) } callback(event) } @@ -163,34 +120,6 @@ func getAddress(toTest string) (string, error) { return toTest, nil } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results -} - func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(r.options.TemplateID), diff --git a/v2/pkg/protocols/others/utils/utils.go b/v2/pkg/protocols/others/utils/utils.go new file mode 100644 index 000000000..804d0a948 --- /dev/null +++ b/v2/pkg/protocols/others/utils/utils.go @@ -0,0 +1,83 @@ +package utils + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// MakeResultEventItemFunc returns a result event for an internal wrapped event item +type MakeResultEventItemFunc func(wrapped *output.InternalWrappedEvent) *output.ResultEvent + +// MakeResultEvent creates a result event from internal wrapped event +func MakeResultEvent(wrapped *output.InternalWrappedEvent, makeEventItemFunc MakeResultEventItemFunc) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := makeEventItemFunc(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := makeEventItemFunc(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := makeEventItemFunc(wrapped) + results = append(results, data) + } + return results +} + +// ExtractFunc performs extracting operation for an extractor on model and returns true or false. +func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJSON(itemStr) + case extractors.XPathExtractor: + return extractor.ExtractHTML(itemStr) + } + return nil +} + +// MatchFunc performs matching operation for a matcher on model and returns true or false. +func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool { + partItem, ok := data[matcher.Part] + if !ok { + return false + } + item := types.ToString(partItem) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(item))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(item)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(item)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(item)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} diff --git a/v2/pkg/protocols/others/wss/wss.go b/v2/pkg/protocols/others/wss/wss.go new file mode 100644 index 000000000..c4004307c --- /dev/null +++ b/v2/pkg/protocols/others/wss/wss.go @@ -0,0 +1 @@ +package wss diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333e..137cdfb62 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -6,8 +6,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" @@ -74,10 +72,6 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool - // Extract performs extracting operation for an extractor on model and returns true or false. - Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 72d880625..bd711492f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -70,7 +70,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestsSSL) == 0 { + if template.Requests() == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -129,6 +129,19 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute return template, nil } +// Requests returns the total number of requests for the template. +func (t *Template) Requests() int { + sum := len(t.RequestsDNS) + + len(t.RequestsHTTP) + + len(t.RequestsFile) + + len(t.RequestsNetwork) + + len(t.RequestsHeadless) + + len(t.Workflows) + + len(t.RequestsSSL) + return sum +} + +// makeRequestsForTemplate compiles all the requests for the template. func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptions) { requests := []protocols.Request{} From 0b11b80d8afb997cb879b702b88300865caa8ccc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 24 Sep 2021 19:35:00 +0530 Subject: [PATCH 004/196] Fixed some bugs with ssl protocols + misc enhancements --- v2/pkg/operators/common/dsl/dsl.go | 8 ++------ v2/pkg/operators/common/dsl/dsl_test.go | 11 +++++++++++ v2/pkg/protocols/others/ssl/ssl.go | 9 +++++++-- v2/pkg/protocols/others/ssl/ssl_test.go | 5 +++++ v2/pkg/protocols/others/utils/utils.go | 2 +- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 0b99505e3..6bff036a1 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -356,12 +356,8 @@ var functions = map[string]govaluate.ExpressionFunction{ }, // is_before_now compares a timestamp and returns true if the first // passed argument is a time.Time that has already passed. - "is_time_before_now": func(args ...interface{}) (interface{}, error) { - if len(args) != 1 { - return nil, ErrDSLArguments - } - isBefore := time.Since(args[0].(time.Time)) > 0 - return isBefore, nil + "time_now": func(args ...interface{}) (interface{}, error) { + return float64(time.Now().Unix()), nil }, } diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index bf2c5bfef..42bf5810d 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -2,7 +2,9 @@ package dsl import ( "testing" + "time" + "github.com/Knetic/govaluate" "github.com/stretchr/testify/require" ) @@ -17,3 +19,12 @@ func TestDSLURLEncodeDecode(t *testing.T) { require.Nil(t, err, "could not url encode") require.Equal(t, "&test\"", decoded, "could not get url decoded data") } + +func TestDSLTimeComparison(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_now() > not_after", HelperFunctions()) + require.Nil(t, err, "could not compare time") + + result, err := compiled.Evaluate(map[string]interface{}{"not_after": float64(time.Now().Unix() - 1000)}) + require.Nil(t, err, "could not evaluate compare time") + require.Equal(t, true, result, "could not get url encoded data") +} diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index 6a33f27e4..ded0274dd 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -65,7 +66,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu if err != nil { return nil } - hostname, _, _ := net.SplitHostPort(input) + hostname, _, _ := net.SplitHostPort(address) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) @@ -81,13 +82,16 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu if !ok { return nil } + r.options.Output.Request(r.options.TemplateID, address, "ssl", err) + gologger.Verbose().Msgf("Sent SSL request to %s", address) + if len(connTLS.ConnectionState().PeerCertificates) == 0 { return nil } data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] data["host"] = input - data["not_after"] = cert.NotAfter + data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) event := &output.InternalWrappedEvent{InternalEvent: data} @@ -116,6 +120,7 @@ func getAddress(toTest string) (string, error) { } else { toTest = parsed.Host } + return toTest, nil } return toTest, nil } diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/others/ssl/ssl_test.go index a043518ab..fef44fbbc 100644 --- a/v2/pkg/protocols/others/ssl/ssl_test.go +++ b/v2/pkg/protocols/others/ssl/ssl_test.go @@ -26,3 +26,8 @@ func TestSSLProtocol(t *testing.T) { err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) require.Nil(t, err, "could not run ssl request") } + +func TestGetAddress(t *testing.T) { + address, _ := getAddress("https://google.com") + require.Equal(t, "google.com:443", address, "could not get correct address") +} diff --git a/v2/pkg/protocols/others/utils/utils.go b/v2/pkg/protocols/others/utils/utils.go index 804d0a948..89771ca90 100644 --- a/v2/pkg/protocols/others/utils/utils.go +++ b/v2/pkg/protocols/others/utils/utils.go @@ -62,7 +62,7 @@ func ExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) m // MatchFunc performs matching operation for a matcher on model and returns true or false. func MatchFunc(data map[string]interface{}, matcher *matchers.Matcher) bool { partItem, ok := data[matcher.Part] - if !ok { + if !ok && len(matcher.DSL) == 0 { return false } item := types.ToString(partItem) From 396f17484ed3eac69e01285c4526339f1465676d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 27 Sep 2021 18:02:49 +0530 Subject: [PATCH 005/196] Added websocket protocol support to nuclei --- v2/go.mod | 4 + v2/go.sum | 9 + .../protocols/others/websocket/websocket.go | 300 ++++++++++++++++++ v2/pkg/protocols/others/wss/wss.go | 1 - v2/pkg/templates/compile.go | 9 +- v2/pkg/templates/templates.go | 4 + 6 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/protocols/others/websocket/websocket.go delete mode 100644 v2/pkg/protocols/others/wss/wss.go diff --git a/v2/go.mod b/v2/go.mod index 3a3e5eb3d..db6b30da5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -79,12 +79,16 @@ require ( github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.1.0 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.2.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 82134aed3..7e8ed14c5 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -143,6 +143,12 @@ github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= +github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -214,6 +220,8 @@ github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= @@ -637,6 +645,7 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go new file mode 100644 index 000000000..db62c6630 --- /dev/null +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -0,0 +1,300 @@ +package websocket + +import ( + "context" + "crypto/tls" + "io" + "net/url" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Inputs contains inputs for the websocket protocol + Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + + // description: | + // Attack is the type of payload combinations to perform. + // + // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // permutations and combinations for all payloads. + // values: + // - "sniper" + // - "pitchfork" + // - "clusterbomb" + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + // description: | + // Payloads contains any payloads for the current request. + // + // Payloads support both key-values combinations where a list + // of payloads is provided, or optionally a single file can also + // be provided as payload which will be read on run-time. + Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + + generator *generators.Generator + attackType generators.Type + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Input is an input for the websocket protocol +type Input struct { + // description: | + // Data is the data to send as the input. + // + // It supports DSL Helper Functions as well as normal expressions. + // examples: + // - value: "\"TEST\"" + // - value: "\"hex_decode('50494e47')\"" + Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + // description: | + // Name is the optional name of the data read to provide matching on. + // examples: + // - value: "\"prefix\"" + Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Payloads) > 0 { + attackType := r.AttackType + if attackType == "" { + attackType = "sniper" + } + r.attackType = generators.StringToType[attackType] + + // Resolve payload paths if they are files. + for name, payload := range r.Payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) + if resolveErr != nil { + return errors.Wrap(resolveErr, "could not read payload file") + } + r.Payloads[name] = final + } + } + r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + if err != nil { + return errors.Wrap(err, "could not parse payloads") + } + } + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + if r.generator != nil { + return r.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + hostname, err := getAddress(input) + if err != nil { + return nil + } + + if r.generator != nil { + iterator := r.generator.NewIterator() + + for { + value, ok := iterator.Value() + if !ok { + break + } + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + } else { + value := make(map[string]interface{}) + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + websocketDialer := ws.Dialer{ + Timeout: time.Duration(r.options.Options.Timeout) * time.Second, + NetDial: r.dialer.Dial, + TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), input) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + + responseBuilder := &strings.Builder{} + if readBuffer != nil { + io.Copy(responseBuilder, readBuffer) // Copy initial response + } + + reqBuilder := &strings.Builder{} + + inputEvents := make(map[string]interface{}) + for _, req := range r.Inputs { + reqBuilder.Grow(len(req.Data)) + reqBuilder.WriteString(req.Data) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + msg, _, err := wsutil.ReadServerData(conn) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + responseBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + if req.Name != "" { + inputEvents[req.Name] = bufferStr + } + + // Run any internal extractors for the request here and add found values to map. + if r.CompiledOperators != nil { + values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, utils.ExtractFunc) + for k, v := range values { + dynamicValues[k] = v + } + } + } + } + r.options.Progress.IncrementRequests() + + if r.options.Options.Debug || r.options.Options.DebugRequests { + requestOutput := reqBuilder.String() + gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", requestOutput) + } + + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) + gologger.Verbose().Msgf("Sent Websocket request to %s", input) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responseOutput) + } + + data := make(map[string]interface{}) + for k, v := range previous { + data[k] = v + } + for k, v := range dynamicValues { + data[k] = v + } + for k, v := range inputEvents { + data[k] = v + } + data["request"] = reqBuilder.String() + data["response"] = responseBuilder.String() + data["host"] = input + data["ip"] = r.dialer.GetDialedIP(hostname) + + event := &output.InternalWrappedEvent{InternalEvent: data} + if r.CompiledOperators != nil { + var ok bool + event.OperatorsResult, ok = r.CompiledOperators.Execute(data, utils.MatchFunc, utils.ExtractFunc) + if ok && event.OperatorsResult != nil { + event.Results = utils.MakeResultEvent(event, r.makeResultEventItem) + } + callback(event) + } + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { + return "", errors.New("invalid websocket provided") + } + parsed, _ := url.Parse(toTest) + if parsed != nil && parsed.Host != "" { + return parsed.Host, nil + } + return "", nil +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + Info: r.options.TemplateInfo, + Type: "websocket", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["responses"]), + } + return data +} diff --git a/v2/pkg/protocols/others/wss/wss.go b/v2/pkg/protocols/others/wss/wss.go deleted file mode 100644 index c4004307c..000000000 --- a/v2/pkg/protocols/others/wss/wss.go +++ /dev/null @@ -1 +0,0 @@ -package wss diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bd711492f..c6c2c24aa 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -137,7 +137,8 @@ func (t *Template) Requests() int { len(t.RequestsNetwork) + len(t.RequestsHeadless) + len(t.Workflows) + - len(t.RequestsSSL) + len(t.RequestsSSL) + + len(t.RequestsWebsocket) return sum } @@ -175,4 +176,10 @@ func makeRequestsForTemplate(template *Template, options protocols.ExecuterOptio } template.Executer = executer.NewExecuter(requests, &options) } + if len(template.RequestsWebsocket) > 0 { + for _, req := range template.RequestsWebsocket { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 6a19d2e52..ecd300fca 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -60,6 +61,9 @@ type Template struct { // description: | // SSL contains the SSL request to make in the template. RequestsSSL []*ssl.Request `yaml:"ssl,omitempty" json:"ssl,omitempty" jsonschema:"title=ssl requests to make,description=SSL requests to make for the template"` + // description: | + // Websocket contains the Websocket request to make in the template. + RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. From b76c1f99cc0898d41ecfe69a8ae7f94a522dab39 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 27 Sep 2021 20:58:05 +0530 Subject: [PATCH 006/196] Misc changes to ws protocol --- .../protocols/others/websocket/websocket.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index db62c6630..4cf3ff741 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "io" + "net/http" "net/url" "strings" "time" @@ -32,6 +33,9 @@ type Request struct { // description: | // Inputs contains inputs for the websocket protocol Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + // description: | + // Origin is the websocket request origin. + Origin string `yaml:"origin,omitempty" jsonschema:"title=origin is the request origin,description=Origin is the websocket request origin"` // description: | // Attack is the type of payload combinations to perform. @@ -163,7 +167,15 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var header ws.HandshakeHeader + + if r.Origin != "" { + header = ws.HandshakeHeaderHTTP(http.Header{ + "Origin": []string{r.Origin}, + }) + } websocketDialer := ws.Dialer{ + Header: header, Timeout: time.Duration(r.options.Options.Timeout) * time.Second, NetDial: r.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, @@ -179,7 +191,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu responseBuilder := &strings.Builder{} if readBuffer != nil { - io.Copy(responseBuilder, readBuffer) // Copy initial response + _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response } reqBuilder := &strings.Builder{} @@ -187,7 +199,6 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu inputEvents := make(map[string]interface{}) for _, req := range r.Inputs { reqBuilder.Grow(len(req.Data)) - reqBuilder.WriteString(req.Data) finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) if dataErr != nil { @@ -195,6 +206,8 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + reqBuilder.WriteString(string(finalData)) + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "websocket", err) @@ -252,6 +265,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu for k, v := range inputEvents { data[k] = v } + data["success"] = "true" data["request"] = reqBuilder.String() data["response"] = responseBuilder.String() data["host"] = input @@ -294,7 +308,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Timestamp: time.Now(), IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), - Response: types.ToString(wrapped.InternalEvent["responses"]), + Response: types.ToString(wrapped.InternalEvent["response"]), } return data } From 7405254c723806ba0bd254ca45adcc15b0fd8db3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 29 Sep 2021 04:53:42 +0530 Subject: [PATCH 007/196] Misc work on ws protocol --- .../protocols/others/websocket/websocket.go | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index 4cf3ff741..c48677f8d 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -30,12 +30,15 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` // description: | // Inputs contains inputs for the websocket protocol Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` // description: | - // Origin is the websocket request origin. - Origin string `yaml:"origin,omitempty" jsonschema:"title=origin is the request origin,description=Origin is the websocket request origin"` + // Headers contains headers for the request. + Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` // description: | // Attack is the type of payload combinations to perform. @@ -167,21 +170,45 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var header ws.HandshakeHeader + header := http.Header{} - if r.Origin != "" { - header = ws.HandshakeHeaderHTTP(http.Header{ - "Origin": []string{r.Origin}, - }) + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + parsed, err := url.Parse(input) + if err != nil { + return errors.Wrap(err, "could not parse input url") + } + payloadValues["Address"] = parsed.Host + payloadValues["Scheme"] = parsed.Scheme + payloadValues["Path"] = parsed.Path + payloadValues["hostname"] = parsed.Hostname() + + for key, value := range r.Headers { + finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + header.Set(key, string(finalData)) } websocketDialer := ws.Dialer{ - Header: header, + Header: ws.HandshakeHeaderHTTP(header), Timeout: time.Duration(r.options.Options.Timeout) * time.Second, NetDial: r.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, } - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), input) + finalAddress, dataErr := expressions.EvaluateByte([]byte(r.Address), payloadValues) + if dataErr != nil { + r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) + r.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), string(finalAddress)) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "ssl", err) r.options.Progress.IncrementFailedRequestsBy(1) @@ -200,7 +227,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu for _, req := range r.Inputs { reqBuilder.Grow(len(req.Data)) - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), dynamicValues) + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) if dataErr != nil { r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) r.options.Progress.IncrementFailedRequestsBy(1) From f8d5d025702da2e9a477b0f04b2602cd2053abc2 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 12 Oct 2021 23:28:24 +0200 Subject: [PATCH 008/196] Adding support to read tcp data stream till the end --- v2/pkg/protocols/network/network.go | 7 +++++ v2/pkg/protocols/network/request.go | 40 ++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index b7de7b920..ef47dc36e 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -60,6 +60,13 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // ReadAll determines if the data stream should be read till the end regardless of the size + // + // Default value for read-all is false. + // examples: + // - value: false + ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index aad3c60ef..ab5176685 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -6,6 +6,7 @@ import ( "io" "net" "net/url" + "os" "strings" "time" @@ -183,13 +184,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin if r.ReadSize != 0 { bufferSize = r.ReadSize } - final := make([]byte, bufferSize) - n, err := conn.Read(final) - if err != nil && err != io.EOF { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - return errors.Wrap(err, "could not read from server") + + var ( + final []byte + n int + ) + + if r.ReadAll { + readInterval := time.After(time.Second * 1) + read_socket: + for { + select { + case <-readInterval: + break read_socket + default: + buf := make([]byte, bufferSize) + nBuf, err := conn.Read(buf) + if err != nil && !os.IsTimeout(err) { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(buf[:nBuf]) + final = append(final, buf...) + n += nBuf + } + } + } else { + final = make([]byte, bufferSize) + n, err = conn.Read(final) + if err != nil && err != io.EOF { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(final[:n]) } - responseBuilder.Write(final[:n]) if r.options.Options.Debug || r.options.Options.DebugResponse { responseOutput := responseBuilder.String() From 14bb1b7b21a48d5bf012a3225367c252016cca03 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Thu, 14 Oct 2021 23:30:37 +0200 Subject: [PATCH 009/196] Implement `-template-url` and `-workflow-url` for retrieving lists of templates/workflows to run. --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/loader.go | 22 ++++-- v2/pkg/catalog/loader/remote_loader.go | 98 ++++++++++++++++++++++++++ v2/pkg/types/types.go | 4 ++ 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/catalog/loader/remote_loader.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 83b13fc6b..81e701fc6 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -56,7 +56,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e87903f14..06abc70ae 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -337,7 +337,9 @@ func (r *Runner) RunEnumeration() error { loaderConfig := loader.Config{ Templates: r.options.Templates, + TemplateURLs: r.options.TemplateURLs, Workflows: r.options.Workflows, + WorkflowURLs: r.options.WorkflowURLs, ExcludeTemplates: r.options.ExcludedTemplates, Tags: r.options.Tags, ExcludeTags: r.options.ExcludeTags, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4ae..6d914ae5c 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -15,7 +15,9 @@ import ( // Config contains the configuration options for the loader type Config struct { Templates []string + TemplateURLs []string Workflows []string + WorkflowURLs []string ExcludeTemplates []string IncludeTemplates []string @@ -37,6 +39,7 @@ type Store struct { pathFilter *filter.PathFilter config *Config finalTemplates []string + finalWorkflows []string templates []*templates.Template workflows []*templates.Template @@ -61,13 +64,24 @@ func New(config *Config) (*Store, error) { IncludedTemplates: config.IncludeTemplates, ExcludedTemplates: config.ExcludeTemplates, }, config.Catalog), + finalTemplates: config.Templates, + finalWorkflows: config.Workflows, + } + + if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) + if err != nil { + return store, err + } + store.finalTemplates = append(store.finalTemplates, remoteTemplates...) + store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...) } // Handle a case with no templates or workflows, where we use base directory - if len(config.Templates) == 0 && len(config.Workflows) == 0 { - config.Templates = append(config.Templates, config.TemplatesDirectory) + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + store.finalTemplates = []string{config.TemplatesDirectory} } - store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil } @@ -90,7 +104,7 @@ func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) { // the complete compiled templates for a nuclei execution configuration. func (store *Store) Load() { store.templates = store.LoadTemplates(store.finalTemplates) - store.workflows = store.LoadWorkflows(store.config.Workflows) + store.workflows = store.LoadWorkflows(store.finalWorkflows) } // ValidateTemplates takes a list of templates and validates them diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go new file mode 100644 index 000000000..ec47ed51b --- /dev/null +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -0,0 +1,98 @@ +package loader + +import ( + "bufio" + "fmt" + "github.com/pkg/errors" + "net/http" + "strings" +) + +type ContentType string + +const ( + Template ContentType = "Template" + Workflow ContentType = "Workflow" +) + +type RemoteContentError struct { + Content []string + Type ContentType + Error error +} + +func getRemoteTemplatesAndWorkflows(templateURLs []string, workflowURLs []string) ([]string, []string, error) { + remoteContentErrorChannel := make(chan RemoteContentError) + + for _, templateURL := range templateURLs { + go getRemoteContent(templateURL, remoteContentErrorChannel, Template) + } + for _, workflowURL := range workflowURLs { + go getRemoteContent(workflowURL, remoteContentErrorChannel, Workflow) + } + + var remoteTemplateList []string + var remoteWorkFlowList []string + var err error + for i := 0; i < (len(templateURLs) + len(workflowURLs)); i++ { + remoteContentError := <-remoteContentErrorChannel + if remoteContentError.Error != nil { + if err != nil { + err = errors.New(remoteContentError.Error.Error() + ": " + err.Error()) + } else { + err = remoteContentError.Error + } + } else { + if remoteContentError.Type == Template { + remoteTemplateList = append(remoteTemplateList, remoteContentError.Content...) + } else if remoteContentError.Type == Workflow { + remoteWorkFlowList = append(remoteWorkFlowList, remoteContentError.Content...) + } + } + } + + return remoteTemplateList, remoteWorkFlowList, err +} + +func getRemoteContent(URL string, w chan<- RemoteContentError, contentType ContentType) { + response, err := http.Get(URL) + if err != nil { + w <- RemoteContentError{ + Error: err, + } + return + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode > 299 { + w <- RemoteContentError{ + Error: fmt.Errorf("get \"%s\": unexpect status %d", URL, response.StatusCode), + } + return + } + + if err != nil { + + } + + scanner := bufio.NewScanner(response.Body) + var templateList []string + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + templateList = append(templateList, text) + } + + if err := scanner.Err(); err != nil { + w <- RemoteContentError{ + Error: errors.Wrap(err, "get \"%s\""), + } + return + } + + w <- RemoteContentError{ + Content: templateList, + Type: contentType, + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dd81c3e4a..b5762a861 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -15,8 +15,12 @@ type Options struct { ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice + // WorkflowURLs specifies URLs to a list of workflows to use + WorkflowURLs goflags.StringSlice // Templates specifies the template/templates to use Templates goflags.StringSlice + // TemplateURLs specifies URLs to a list of templates to use + TemplateURLs goflags.StringSlice // ExcludedTemplates specifies the template/templates to exclude ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request. From 81102750a1b384baefb195101be8c23e5f83c414 Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Thu, 14 Oct 2021 23:30:51 +0200 Subject: [PATCH 010/196] Implement integration tests for remote template and workflow urls. --- integration_tests/loader/basic.yaml | 10 ++ .../loader/condition-matched.yaml | 11 ++ integration_tests/loader/get-headers.yaml | 17 +++ integration_tests/loader/get.yaml | 15 +++ integration_tests/loader/template-list.yaml | 2 + integration_tests/loader/workflow-list.yaml | 2 + v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/loader.go | 116 ++++++++++++++++++ v2/internal/testutils/integration.go | 21 +++- 9 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 integration_tests/loader/basic.yaml create mode 100644 integration_tests/loader/condition-matched.yaml create mode 100644 integration_tests/loader/get-headers.yaml create mode 100644 integration_tests/loader/get.yaml create mode 100644 integration_tests/loader/template-list.yaml create mode 100644 integration_tests/loader/workflow-list.yaml create mode 100644 v2/cmd/integration-test/loader.go diff --git a/integration_tests/loader/basic.yaml b/integration_tests/loader/basic.yaml new file mode 100644 index 000000000..f49193b18 --- /dev/null +++ b/integration_tests/loader/basic.yaml @@ -0,0 +1,10 @@ +id: workflow-example + +info: + name: Test Workflow Template + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/condition-matched.yaml b/integration_tests/loader/condition-matched.yaml new file mode 100644 index 000000000..8b0a65732 --- /dev/null +++ b/integration_tests/loader/condition-matched.yaml @@ -0,0 +1,11 @@ +id: condition-matched-workflow + +info: + name: Condition Matched Workflow + author: pdteam + severity: info + +workflows: + - template: workflow/match-1.yaml + subtemplates: + - template: workflow/match-2.yaml \ No newline at end of file diff --git a/integration_tests/loader/get-headers.yaml b/integration_tests/loader/get-headers.yaml new file mode 100644 index 000000000..bae367052 --- /dev/null +++ b/integration_tests/loader/get-headers.yaml @@ -0,0 +1,17 @@ +id: basic-get-headers + +info: + name: Basic GET Headers Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + test: nuclei + matchers: + - type: word + words: + - "This is test headers matcher text" \ No newline at end of file diff --git a/integration_tests/loader/get.yaml b/integration_tests/loader/get.yaml new file mode 100644 index 000000000..c7e07e8cf --- /dev/null +++ b/integration_tests/loader/get.yaml @@ -0,0 +1,15 @@ +id: basic-get + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "This is test matcher text" \ No newline at end of file diff --git a/integration_tests/loader/template-list.yaml b/integration_tests/loader/template-list.yaml new file mode 100644 index 000000000..fae00d6ce --- /dev/null +++ b/integration_tests/loader/template-list.yaml @@ -0,0 +1,2 @@ +loader/get.yaml +loader/get-headers.yaml diff --git a/integration_tests/loader/workflow-list.yaml b/integration_tests/loader/workflow-list.yaml new file mode 100644 index 000000000..3f56730e7 --- /dev/null +++ b/integration_tests/loader/workflow-list.yaml @@ -0,0 +1,2 @@ +loader/basic.yaml +loader/condition-matched.yaml diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index baa743e8d..94e6c2159 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -26,6 +26,7 @@ func main() { "network": networkTestcases, "dns": dnsTestCases, "workflow": workflowTestcases, + "loader": loaderTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go new file mode 100644 index 000000000..13aa3f05e --- /dev/null +++ b/v2/cmd/integration-test/loader.go @@ -0,0 +1,116 @@ +package main + +import ( + "fmt" + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "net/http" + "net/http/httptest" + "os" + "strings" +) + +var loaderTestcases = map[string]testutils.TestCase{ + "loader/template-list.yaml": &remoteTemplateList{}, + "loader/workflow-list.yaml": &remoteWorkflowList{}, + "loader/nonexistent-template-list.yaml": &nonExistentTemplateList{}, + "loader/nonexistent-workflow-list.yaml": &nonExistentWorkflowList{}, +} + +type remoteTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteTemplateList) Execute(templateList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(templateList) + if err != nil { + w.WriteHeader(500) + } + w.Write(file) + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/template_list") + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} + +type remoteWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *remoteWorkflowList) Execute(workflowList string) error { + router := httprouter.New() + + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + if strings.EqualFold(r.Header.Get("test"), "nuclei") { + fmt.Fprintf(w, "This is test headers matcher text") + } + })) + + router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + file, err := os.ReadFile(workflowList) + if err != nil { + w.WriteHeader(500) + } + w.Write(file) + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/workflow_list") + if err != nil { + return err + } + if len(results) != 3 { + return errIncorrectResultsCount(results) + } + return nil +} + +type nonExistentTemplateList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentTemplateList) Execute(nonExistingTemplateList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-tu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} + +type nonExistentWorkflowList struct{} + +// Execute executes a test case and returns an error if occurred +func (h *nonExistentWorkflowList) Execute(nonExistingWorkflowList string) error { + router := httprouter.New() + ts := httptest.NewServer(router) + defer ts.Close() + + _, err := testutils.RunNucleiBareArgsAndGetResults(debug, "-target", ts.URL, "-wu", ts.URL+"/404") + if err == nil { + return fmt.Errorf("expected error for nonexisting workflow url") + } + + return nil +} diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index f2c398869..4435b957f 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -12,15 +12,15 @@ import ( // RunNucleiTemplateAndGetResults returns a list of results for a template func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(true, template, url, debug, extra...) + return RunNucleiAndGetResults(true, template, url, debug, extra...) } // RunNucleiWorkflowAndGetResults returns a list of results for a workflow func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - return runNucleiAndGetResults(false, template, url, debug, extra...) + return RunNucleiAndGetResults(false, template, url, debug, extra...) } -func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { +func RunNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { var templateOrWorkflowFlag string if isTemplate { templateOrWorkflowFlag = "-t" @@ -28,11 +28,22 @@ func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, e templateOrWorkflowFlag = "-w" } - cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent") + return RunNucleiBareArgsAndGetResults(debug, append([]string{ + templateOrWorkflowFlag, + template, + "-target", + url, + }, extra...)...) +} + +func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("./nuclei") if debug { - cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug") + cmd.Args = append(cmd.Args, "-debug") cmd.Stderr = os.Stderr fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") } cmd.Args = append(cmd.Args, extra...) data, err := cmd.Output() From 5584fc285d6377bc0fb53bf14385a1f7b09bef10 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Tue, 19 Oct 2021 00:58:49 +0300 Subject: [PATCH 011/196] Add regexp pattern for template-id --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491ac..0b925d7e7 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"` // description: | // Info contains metadata information about the template. // examples: From 0ba3b19f1f7653a41327315644b7fcccb57de1c2 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 19 Oct 2021 11:31:36 +0200 Subject: [PATCH 012/196] fixing merge issues --- v2/pkg/protocols/network/request.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 399699a0f..a1425ebae 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -191,13 +191,13 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.ReadSize != 0 { bufferSize = request.ReadSize } - - var ( + + var ( final []byte n int ) - if r.ReadAll { + if request.ReadAll { readInterval := time.After(time.Second * 1) read_socket: for { @@ -208,7 +208,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - r.options.Output.Request(r.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplateID, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(buf[:nBuf]) From ae99a88f52b4ddb260818a8ba80a28f650233894 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 19 Oct 2021 19:39:33 +0200 Subject: [PATCH 013/196] timer.after => timer --- v2/pkg/protocols/network/request.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index a1425ebae..6beecc6e1 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -198,17 +198,25 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input ) if request.ReadAll { - readInterval := time.After(time.Second * 1) + readInterval := time.NewTimer(time.Second * 1) + // stop the timer and drain the channel + closeTimer := func(t *time.Timer) { + if !t.Stop() { + <-t.C + } + } read_socket: for { select { - case <-readInterval: + case <-readInterval.C: + closeTimer(readInterval) break read_socket default: buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { request.options.Output.Request(request.options.TemplateID, address, "network", err) + closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(buf[:nBuf]) From a1e099493dc3448465eccb7ee3072487fd03403f Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 00:43:33 +0300 Subject: [PATCH 014/196] Use simpler regexp for template id validation --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 0b925d7e7..a9a0e14ea 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^(?:[a-zA-Z0-9][a-zA-Z0-9]*-?[a-zA-Z0-9]+?)+$"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"` // description: | // Info contains metadata information about the template. // examples: From cbc5192e9a3e1d7f5409385ef26885d0154c1c6e Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 20 Oct 2021 00:02:06 +0200 Subject: [PATCH 015/196] Adding proxy socks support to headless browser --- .../protocols/headless/engine/http_client.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d541..9da2aaa93 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -1,13 +1,17 @@ package engine import ( + "context" "crypto/tls" + "fmt" + "net" "net/http" "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout @@ -28,6 +32,22 @@ func newhttpClient(options *types.Options) *http.Client { if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } + } else if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } } return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} From 9e0144b6aede383452969ae81a771d231dfdd6e2 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 01:31:38 +0300 Subject: [PATCH 016/196] Add template-id validation --- v2/pkg/parsers/parser.go | 30 ++++++++++------ v2/pkg/parsers/parser_test.go | 68 +++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 v2/pkg/parsers/parser_test.go diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index b65cb79e2..1d5213ae3 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -17,7 +17,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) -const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" +const ( + mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" + invalidFieldFormatTemplate = "invalid field format for '%s' (allowed format is %s)" +) // LoadTemplate returns true if the template is valid and matches the filtering criteria. func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) { @@ -30,12 +33,11 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, nil } - templateInfo := template.Info - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -45,10 +47,8 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, templateParseError } - templateInfo := template.Info - if len(template.Workflows) > 0 { - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } return true, nil @@ -71,10 +71,8 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode return match, err } -func validateMandatoryInfoFields(info *model.Info) error { - if info == nil { - return fmt.Errorf(mandatoryFieldMissingTemplate, "info") - } +func validateTemplateFields(template *templates.Template) error { + info := template.Info if utils.IsBlank(info.Name) { return fmt.Errorf(mandatoryFieldMissingTemplate, "name") @@ -83,6 +81,15 @@ func validateMandatoryInfoFields(info *model.Info) error { if info.Authors.IsEmpty() { return fmt.Errorf(mandatoryFieldMissingTemplate, "author") } + + if template.ID == "" { + return fmt.Errorf(mandatoryFieldMissingTemplate, "id") + } + + if !templateIDRegexp.MatchString(template.ID) { + return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()) + } + return nil } @@ -90,6 +97,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) + templateIDRegexp = regexp.MustCompile(`^[A-Za-z0-9-_]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go new file mode 100644 index 000000000..e1e2a1282 --- /dev/null +++ b/v2/pkg/parsers/parser_test.go @@ -0,0 +1,68 @@ +package parsers + +import ( + "errors" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/stretchr/testify/require" +) + +func TestLoadTemplate(t *testing.T) { + origTemplatesCache := parsedTemplatesCache + defer func() { parsedTemplatesCache = origTemplatesCache }() + + tt := []struct { + name string + template *templates.Template + templateErr error + + expectedErr error + }{ + { + name: "valid", + template: &templates.Template{ + ID: "CVE-2021-27330", + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + }, + { + name: "missingName", + template: &templates.Template{}, + expectedErr: errors.New("mandatory 'name' field is missing"), + }, + { + name: "invalidID", + template: &templates.Template{ + ID: "ABC DEF", + Info: model.Info{ + Name: "Invalid ID", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(tc.name, tagFilter, nil) + if tc.expectedErr == nil { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, tc.expectedErr, err) + require.False(t, success) + } + }) + } +} From df12b0d2e6ac754aa5691addf0c327af2a6ae0b3 Mon Sep 17 00:00:00 2001 From: kchason Date: Tue, 19 Oct 2021 23:06:02 -0400 Subject: [PATCH 017/196] Add CLI options for passing client cert auth --- v2/cmd/nuclei/main.go | 3 +++ v2/pkg/types/types.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 23fd7b1db..c474615a2 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -105,6 +105,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), + flagSet.StringVar(&options.ClientCertFile, "client-cert", "cc", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVar(&options.ClientKeyFile, "client-key", "ck", "client key file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVar(&options.ClientCAFile, "client-ca", "ca", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), ) createGroup(flagSet, "interactsh", "interactsh", diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dafb2f3e7..16bb81694 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -156,4 +156,10 @@ type Options struct { NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool + // ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts + ClientCertFile string + // ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts + ClientKeyFile string + // ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + ClientCAFile string } From e6728e8ff9f4d1d25d91b0f9d047dbca84010308 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 20 Oct 2021 13:26:47 +0200 Subject: [PATCH 018/196] Making headless httpclient more similar to real browsers --- v2/pkg/protocols/headless/engine/http_client.go | 15 ++++++++++++++- v2/pkg/protocols/headless/engine/rules.go | 3 +++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d541..94c33563c 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -3,6 +3,7 @@ package engine import ( "crypto/tls" "net/http" + "net/http/cookiejar" "net/url" "time" @@ -30,5 +31,17 @@ func newhttpClient(options *types.Options) *http.Client { } } - return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} + jar, _ := cookiejar.New(nil) + + httpclient := &http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout*3) * time.Second, + Jar: jar, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // the browser should follow redirects not us + return http.ErrUseLastResponse + }, + } + + return httpclient } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a254902eb..a802a64a9 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,6 +8,9 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { + // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) + for _, rule := range p.rules { if rule.Part != "request" { continue From e2052dedc1f9abd0316ed9257bdd92f9e7e56dac Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 20 Oct 2021 09:44:14 -0400 Subject: [PATCH 019/196] Add options to readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 17d7d211c..823f3a21c 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,9 @@ CONFIGURATIONS: -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode -env-vars enable environment variables support + -cc -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck -client-key client key file (PEM-encoded) used for authenticating against scanned hosts + -ca -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: -no-interactsh disable interactsh server for OOB testing From 9c77f15012d87d9126fb44096e935c309e998ad4 Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 20 Oct 2021 11:32:09 -0400 Subject: [PATCH 020/196] Argument checks for presence and validity --- v2/cmd/nuclei/main.go | 6 +++--- v2/internal/runner/options.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c474615a2..c4e76e2b5 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -105,9 +105,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), - flagSet.StringVar(&options.ClientCertFile, "client-cert", "cc", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), - flagSet.StringVar(&options.ClientKeyFile, "client-key", "ck", "client key file (PEM-encoded) used for authenticating against scanned hosts"), - flagSet.StringVar(&options.ClientCAFile, "client-ca", "ca", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), + flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), ) createGroup(flagSet, "interactsh", "interactsh", diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index cba1af5bf..fd43722bc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -98,6 +98,14 @@ func validateOptions(options *types.Options) error { validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) } + // Verify if any of the client certificate options were set since it requires all three to work properly + if len(options.ClientCertFile) > 0 || len(options.ClientKeyFile) > 0 || len(options.ClientCAFile) > 0 { + if len(options.ClientCertFile) == 0 || len(options.ClientKeyFile) == 0 || len(options.ClientCAFile) == 0 { + return errors.New("if a client certification option is provided, then all three must be provided") + } + validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) + } + return nil } @@ -174,3 +182,14 @@ func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPat } } } + +func validateCertificatePaths(certificatePaths []string) { + for _, certificatePath := range certificatePaths { + if _, err := os.Stat(certificatePath); os.IsNotExist(err) { + // The provided path to the PEM certificate does not exist for the client authentication. As this is + // required for successful authentication, log and return an error + gologger.Fatal().Msgf("The given path (%s) to the certificate does not exist!", certificatePath) + break + } + } +} From f1cd0a5d288de0fc4212287ec1f0a06033e9fc7b Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 22:58:36 +0300 Subject: [PATCH 021/196] Update template id regexp --- v2/pkg/parsers/parser.go | 2 +- v2/pkg/parsers/parser_test.go | 2 +- v2/pkg/templates/templates.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 1d5213ae3..7bde724f0 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -97,7 +97,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) - templateIDRegexp = regexp.MustCompile(`^[A-Za-z0-9-_]+$`) + templateIDRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index e1e2a1282..fefde3237 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -46,7 +46,7 @@ func TestLoadTemplate(t *testing.T) { Authors: stringslice.StringSlice{Value: "Author"}, }, }, - expectedErr: errors.New("invalid field format for 'id' (allowed format is ^[A-Za-z0-9-_]+$)"), + expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), }, } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index a9a0e14ea..8f5a4e167 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,7 +27,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^[A-Za-z0-9-_]+$"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: From 5d0f6b2622f54aa8f1980f5ef83138523e23c26a Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 23:14:04 +0300 Subject: [PATCH 022/196] Improve invalid template id tests --- v2/pkg/parsers/parser_test.go | 54 ++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index fefde3237..e94b2ee30 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -2,6 +2,7 @@ package parsers import ( "errors" + "fmt" "testing" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" @@ -37,17 +38,6 @@ func TestLoadTemplate(t *testing.T) { template: &templates.Template{}, expectedErr: errors.New("mandatory 'name' field is missing"), }, - { - name: "invalidID", - template: &templates.Template{ - ID: "ABC DEF", - Info: model.Info{ - Name: "Invalid ID", - Authors: stringslice.StringSlice{Value: "Author"}, - }, - }, - expectedErr: errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), - }, } for _, tc := range tt { @@ -65,4 +55,46 @@ func TestLoadTemplate(t *testing.T) { } }) } + + t.Run("invalidTemplateID", func(t *testing.T) { + tt := []struct { + id string + success bool + }{ + {id: "A-B-C", success: true}, + {id: "A-B-C-1", success: true}, + {id: "CVE_2021_27330", success: true}, + {id: "ABC DEF", success: false}, + {id: "_-__AAA_", success: false}, + {id: " CVE-2021-27330", success: false}, + {id: "CVE-2021-27330 ", success: false}, + {id: "CVE-2021-27330-", success: false}, + {id: "-CVE-2021-27330-", success: false}, + {id: "CVE-2021--27330", success: false}, + {id: "CVE-2021+27330", success: false}, + } + for i, tc := range tt { + name := fmt.Sprintf("regexp%d", i) + t.Run(name, func(t *testing.T) { + template := &templates.Template{ + ID: tc.id, + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + } + parsedTemplatesCache.Store(name, template, nil) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(name, tagFilter, nil) + if tc.success { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err) + require.False(t, success) + } + }) + } + }) } From ff7a5997a225a81ed35b9017b76105ffe186111f Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 20 Oct 2021 23:24:11 +0300 Subject: [PATCH 023/196] Return multiple errors in template validation --- v2/pkg/parsers/parser.go | 15 ++++++++++----- v2/pkg/parsers/parser_test.go | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 7bde724f0..83059baa7 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "gopkg.in/yaml.v2" @@ -74,20 +75,24 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode func validateTemplateFields(template *templates.Template) error { info := template.Info + var errors []string + if utils.IsBlank(info.Name) { - return fmt.Errorf(mandatoryFieldMissingTemplate, "name") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name")) } if info.Authors.IsEmpty() { - return fmt.Errorf(mandatoryFieldMissingTemplate, "author") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author")) } if template.ID == "" { - return fmt.Errorf(mandatoryFieldMissingTemplate, "id") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id")) + } else if !templateIDRegexp.MatchString(template.ID) { + errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())) } - if !templateIDRegexp.MatchString(template.ID) { - return fmt.Errorf(invalidFieldFormatTemplate, "id", templateIDRegexp.String()) + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, ", ")) } return nil diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index e94b2ee30..ef74a317e 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -34,9 +34,19 @@ func TestLoadTemplate(t *testing.T) { }, }, { - name: "missingName", + name: "emptyTemplate", template: &templates.Template{}, - expectedErr: errors.New("mandatory 'name' field is missing"), + expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"), + }, + { + name: "emptyNameWithInvalidID", + template: &templates.Template{ + ID: "invalid id", + Info: model.Info{ + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), }, } From fdb7c0d6cda8b2acf11af247d9eaf65f5b58f036 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Oct 2021 06:20:40 +0530 Subject: [PATCH 024/196] dev update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 8a996b62d..91b153cf2 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.3` +const Version = `2.5.4-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 450af73eee9e6f2a8444fb42e8f6983e6453e312 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:44:51 -0400 Subject: [PATCH 025/196] Fix README spacing with commas --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 823f3a21c..37e2b745d 100644 --- a/README.md +++ b/README.md @@ -128,9 +128,9 @@ CONFIGURATIONS: -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode -env-vars enable environment variables support - -cc -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck -client-key client key file (PEM-encoded) used for authenticating against scanned hosts - -ca -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -cc, -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: -no-interactsh disable interactsh server for OOB testing From 0a57a1aa4d01095bd936fe3d64df3b73c19351fb Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:48:13 -0400 Subject: [PATCH 026/196] Client certificate authentication for headless connections --- .../protocols/headless/engine/http_client.go | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index c9ec6e0ce..eed7875a3 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -2,6 +2,9 @@ package engine import ( "crypto/tls" + "crypto/x509" + "io/ioutil" + "log" "net/http" "time" @@ -12,15 +15,40 @@ import ( // newhttpClient creates a new http client for headless communication with a timeout func newhttpClient(options *types.Options) *http.Client { dialer := protocolstate.Dialer + + // Set the base TLS configuration definition + tlsConfig := &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + } + + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + transport := &http.Transport{ DialContext: dialer.Dial, MaxIdleConns: 500, MaxIdleConnsPerHost: 500, MaxConnsPerHost: 500, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, + TLSClientConfig: tlsConfig, } return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} } From f5b9eb32a1c5a3727fe21d47ac1af5083207afb2 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 13:54:56 -0400 Subject: [PATCH 027/196] Client certificate authentication for pooled connections --- .../http/httpclientpool/clientpool.go | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index a4faef07d..3b0ce9b48 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -3,7 +3,10 @@ package httpclientpool import ( "context" "crypto/tls" + "crypto/x509" "fmt" + "io/ioutil" + "log" "net" "net/http" "net/http/cookiejar" @@ -161,16 +164,40 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl disableKeepAlives = configuration.Connection.DisableKeepAlive } + // Set the base TLS configuration definition + tlsConfig := &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + } + + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + transport := &http.Transport{ DialContext: Dialer.Dial, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, MaxConnsPerHost: maxConnsPerHost, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - DisableKeepAlives: disableKeepAlives, + TLSClientConfig: tlsConfig, + DisableKeepAlives: disableKeepAlives, } // Attempts to overwrite the dial function with the socks proxied version From 18deddb07abcabc056c81f19a4149f0de0449807 Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 14:10:47 -0400 Subject: [PATCH 028/196] Missing error package --- v2/pkg/protocols/http/request.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1a619c179..44793cdc6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,6 +2,7 @@ package http import ( "bytes" + "errors" "fmt" "io" "io/ioutil" From 99492911a36adb869e4ab17e5426084250b5f75a Mon Sep 17 00:00:00 2001 From: kchason Date: Thu, 21 Oct 2021 14:21:52 -0400 Subject: [PATCH 029/196] Revert "Missing error package" This reverts commit 18deddb07abcabc056c81f19a4149f0de0449807. --- v2/pkg/protocols/http/request.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 44793cdc6..1a619c179 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,7 +2,6 @@ package http import ( "bytes" - "errors" "fmt" "io" "io/ioutil" From a911245d26a11f6a3f9aef61f6e182cbfe131d7f Mon Sep 17 00:00:00 2001 From: kchason Date: Sun, 24 Oct 2021 22:53:28 -0400 Subject: [PATCH 030/196] Re-add short command for env-vars lost during merge conflict resolution --- README.md | 2 +- v2/cmd/nuclei/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 757264fbf..ca1988816 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ CONFIGURATIONS: -r, -resolvers string file containing resolver list for nuclei -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support to be used in template + -ev, -env-vars enable environment variables support to be used in template -cc, -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts -ck, -client-key client key file (PEM-encoded) used for authenticating against scanned hosts -ca, -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 43b9608c5..1b898f68e 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -92,7 +92,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), - flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables to be used in template"), + flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"), flagSet.StringVarP(&options.ClientCertFile, "client-cert", "cc", "", "client certificate file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientKeyFile, "client-key", "ck", "", "client key file (PEM-encoded) used for authenticating against scanned hosts"), flagSet.StringVarP(&options.ClientCAFile, "client-ca", "ca", "", "client certificate authority file (PEM-encoded) used for authenticating against scanned hosts"), From 1f8a9474cfa07682dc7e38a897b5b938a10a83f0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 25 Oct 2021 12:00:09 +0000 Subject: [PATCH 031/196] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25 12:00:09 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 25 ++++++++++++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 35 ++++++++++++++++++------------- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 30ed998fb..a3fffe351 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2392,6 +2392,31 @@ read-size: 2048 ``` + + +
+ +
+ +read-all bool + +
+
+ +ReadAll determines if the data stream should be read till the end regardless of the size + +Default value for read-all is false. + + + +Examples: + + +```yaml +read-all: false +``` + +
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 56253b7ed..57f8fc7fa 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -809,6 +809,11 @@ "title": "size of network response to read", "description": "Size of response to read at the end. Default is 1024 bytes" }, + "read-all": { + "type": "boolean", + "title": "read all response stream", + "description": "Read all response stream till the server stops sending" + }, "matchers": { "items": { "$ref": "#/definitions/matchers.Matcher" diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 0b19e675d..d6042dfbd 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -843,7 +843,7 @@ func init() { FieldName: "network", }, } - NETWORKRequestDoc.Fields = make([]encoder.Doc, 9) + NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" NETWORKRequestDoc.Fields[0].Note = "" @@ -883,22 +883,29 @@ func init() { NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end" NETWORKRequestDoc.Fields[5].AddExample("", 2048) - NETWORKRequestDoc.Fields[6].Name = "matchers" - NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher" + NETWORKRequestDoc.Fields[6].Name = "read-all" + NETWORKRequestDoc.Fields[6].Type = "bool" NETWORKRequestDoc.Fields[6].Note = "" - NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." - NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" - NETWORKRequestDoc.Fields[7].Name = "extractors" - NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor" + NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false." + NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size" + + NETWORKRequestDoc.Fields[6].AddExample("", false) + NETWORKRequestDoc.Fields[7].Name = "matchers" + NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher" NETWORKRequestDoc.Fields[7].Note = "" - NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." - NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" - NETWORKRequestDoc.Fields[8].Name = "matchers-condition" - NETWORKRequestDoc.Fields[8].Type = "string" + NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + NETWORKRequestDoc.Fields[8].Name = "extractors" + NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor" NETWORKRequestDoc.Fields[8].Note = "" - NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Values = []string{ + NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + NETWORKRequestDoc.Fields[9].Name = "matchers-condition" + NETWORKRequestDoc.Fields[9].Type = "string" + NETWORKRequestDoc.Fields[9].Note = "" + NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Values = []string{ "and", "or", } From 4a59c3dd84d034c0711e4b480fbd7cc64657f1bb Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 25 Oct 2021 12:09:59 +0000 Subject: [PATCH 032/196] Auto Generate Syntax Docs + JSONSchema [Mon Oct 25 12:09:59 UTC 2021] :robot: --- nuclei-jsonschema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 57f8fc7fa..1db207d77 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -850,6 +850,7 @@ ], "properties": { "id": { + "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "type": "string", "title": "id of the template", "description": "The Unique ID for the template", From ef3397b84de7a49569d2893700689de515a5715d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 17:42:01 +0530 Subject: [PATCH 033/196] Mark error for template syntax warning --- v2/pkg/parsers/parser.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 83059baa7..7cd0c7b99 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -35,6 +35,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] } if validationError := validateTemplateFields(template); validationError != nil { + stats.Increment(SyntaxErrorStats) return false, validationError } From ca6cc513021d2e6a85aeb879055bc158c482fea8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 17:47:39 +0530 Subject: [PATCH 034/196] Color change for warning disclaimer --- v2/internal/runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index b75f09c33..56bf4ea4b 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") - gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") + gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } From c6445519ecee46b0195a499d6746bb0a6bf372af Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:19:27 +0530 Subject: [PATCH 035/196] Fixed bug with github client and paths with no slash --- v2/pkg/reporting/trackers/github/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 672f62ede..8d6cdb864 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -58,6 +58,9 @@ func New(options *Options) (*Integration, error) { if err != nil { return nil, errors.Wrap(err, "could not parse custom baseurl") } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } client.BaseURL = parsed } return &Integration{client: client, options: options}, nil From 4a29443752b9f158cab15359dc09a3691262fca5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:22:33 +0530 Subject: [PATCH 036/196] fix panic: Only generate curl command if request is not nil --- v2/pkg/protocols/http/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1a619c179..3451e5f3d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -384,7 +384,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate }() var curlCommand string - if !request.Unsafe && resp != nil && generatedRequest.request != nil { + if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil { bodyBytes, _ := generatedRequest.request.BodyBytes() resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) command, _ := http2curl.GetCurlCommand(resp.Request) From bfaea5b89d39f529b8dbf421e11fe2c61403df5c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 18:35:45 +0530 Subject: [PATCH 037/196] Added more variables for DNS requests --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/protocols/dns/dns.go | 20 +++++++++++++++++++- v2/pkg/protocols/dns/dns_test.go | 11 +++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index e5e74498f..5e6e5f0c7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -53,6 +53,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 + github.com/weppos/publicsuffix-go v0.15.0 github.com/xanzy/go-gitlab v0.50.3 github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index b95089c7e..a497178f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -750,6 +750,8 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ= +github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 15f42dc02..b5c27f7f6 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -7,6 +7,8 @@ import ( "github.com/miekg/dns" "github.com/pkg/errors" + "github.com/weppos/publicsuffix-go/publicsuffix" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -132,7 +134,7 @@ func (request *Request) Make(domain string) (*dns.Msg, error) { var q dns.Question - final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain}) + final := replacer.Replace(request.Name, generateDNSVariables(domain)) q.Name = dns.Fqdn(final) q.Qclass = request.class @@ -198,3 +200,19 @@ func classToInt(class string) uint16 { } return uint16(result) } + +func generateDNSVariables(domain string) map[string]interface{} { + parsed, err := publicsuffix.Parse(domain) + if err != nil { + return map[string]interface{}{"FQDN": domain} + } + + domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".") + return map[string]interface{}{ + "{{FQDN}}": domain, + "{{RDN}}": domainName, + "{{DN}}": parsed.SLD, + "{{TLD}}": parsed.TLD, + "{{SD}}": parsed.TRD, + } +} diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index ae287984d..74b384025 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -10,6 +10,17 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" ) +func TestGenerateDNSVariables(t *testing.T) { + vars := generateDNSVariables("www.projectdiscovery.io") + require.Equal(t, map[string]interface{}{ + "{{FQDN}}": "www.projectdiscovery.io", + "{{RDN}}": "projectdiscovery.io", + "{{DN}}": "projectdiscovery", + "{{TLD}}": "io", + "{{SD}}": "www", + }, vars, "could not get dns variables") +} + func TestDNSCompileMake(t *testing.T) { options := testutils.DefaultOptions From 09a2d88364c5cb5a36a242e28b5640d424ca055f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 19:33:37 +0530 Subject: [PATCH 038/196] Misc --- v2/pkg/protocols/dns/dns.go | 10 +++++----- v2/pkg/protocols/dns/dns_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index b5c27f7f6..66ed617aa 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -209,10 +209,10 @@ func generateDNSVariables(domain string) map[string]interface{} { domainName := strings.Join([]string{parsed.SLD, parsed.TLD}, ".") return map[string]interface{}{ - "{{FQDN}}": domain, - "{{RDN}}": domainName, - "{{DN}}": parsed.SLD, - "{{TLD}}": parsed.TLD, - "{{SD}}": parsed.TRD, + "FQDN": domain, + "RDN": domainName, + "DN": parsed.SLD, + "TLD": parsed.TLD, + "SD": parsed.TRD, } } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index 74b384025..85b00bd94 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -13,11 +13,11 @@ import ( func TestGenerateDNSVariables(t *testing.T) { vars := generateDNSVariables("www.projectdiscovery.io") require.Equal(t, map[string]interface{}{ - "{{FQDN}}": "www.projectdiscovery.io", - "{{RDN}}": "projectdiscovery.io", - "{{DN}}": "projectdiscovery", - "{{TLD}}": "io", - "{{SD}}": "www", + "FQDN": "www.projectdiscovery.io", + "RDN": "projectdiscovery.io", + "DN": "projectdiscovery", + "TLD": "io", + "SD": "www", }, vars, "could not get dns variables") } From ba108580c6fc539b372f224ff8a0b67a57a00db1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 19:36:37 +0530 Subject: [PATCH 039/196] Fixed fqdn issue with tld parser --- v2/pkg/protocols/dns/dns.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 66ed617aa..8aab88af2 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -202,7 +202,7 @@ func classToInt(class string) uint16 { } func generateDNSVariables(domain string) map[string]interface{} { - parsed, err := publicsuffix.Parse(domain) + parsed, err := publicsuffix.Parse(strings.TrimSuffix(domain, ".")) if err != nil { return map[string]interface{}{"FQDN": domain} } From 6541b04f4c5a2d8e74ae1f0c5282033065cd9f9d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 23:24:42 +0530 Subject: [PATCH 040/196] Added new type and exclude-type flag --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/filter/tag_filter.go | 39 ++++++++++++++- .../catalog/loader/filter/tag_filter_test.go | 49 +++++++++++++------ v2/pkg/catalog/loader/loader.go | 4 ++ v2/pkg/parsers/parser.go | 6 +-- v2/pkg/templates/templates.go | 18 +++++++ v2/pkg/types/types.go | 4 ++ 8 files changed, 104 insertions(+), 20 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..99ef65582 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,6 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f806952..c4c550ea4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,6 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, + Types: r.options.Protocols, + ExcludeTypes: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index ed8722c90..c33e96fd7 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -15,6 +15,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} + types map[string]struct{} + excludeTypes map[string]struct{} } // ErrExcluded is returned for excluded templates @@ -25,7 +27,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -51,6 +53,9 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa return false, nil } + if !isTemplateTypeMatch(tagFilter, templateType) { + return false, nil + } return true, nil } @@ -116,6 +121,24 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } +func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { + if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { + return true + } + + included := true + if len(tagFilter.types) > 0 { + _, included = tagFilter.types[templateType] + } + + excluded := false + if len(tagFilter.excludeTypes) > 0 { + _, excluded = tagFilter.excludeTypes[templateType] + } + + return included && !excluded +} + type Config struct { Tags []string ExcludeTags []string @@ -123,6 +146,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string + Types []string + ExcludeTypes []string } // New returns a tag filter for nuclei tag based execution @@ -136,6 +161,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), + types: make(map[string]struct{}), + excludeTypes: make(map[string]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -177,6 +204,16 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } + for _, tag := range config.Types { + if _, ok := filter.types[tag]; !ok { + filter.types[tag] = struct{}{} + } + } + for _, tag := range config.ExcludeTypes { + if _, ok := filter.excludeTypes[tag]; !ok { + filter.excludeTypes[tag] = struct{}{} + } + } return filter } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 22d18b189..7599c27b0 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -15,19 +15,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.False(t, matched, "could not get correct match") }) } @@ -36,7 +36,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +46,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +55,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +63,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +88,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,16 +97,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil) + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + require.False(t, matched, "could not get correct match") + }) + t.Run("match-type", func(t *testing.T) { + filter := New(&Config{ + Types: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + require.True(t, matched, "could not get correct match") + }) + t.Run("match-exclude-type", func(t *testing.T) { + filter := New(&Config{ + ExcludeTypes: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + require.True(t, matched, "could not get correct match") + + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4ae..fe725e2ee 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -21,6 +21,8 @@ type Config struct { Tags []string ExcludeTags []string + Types []string + ExcludeTypes []string Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -56,6 +58,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, + Types: config.Types, + ExcludeTypes: config.ExcludeTypes, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 7cd0c7b99..9e201fb08 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -39,7 +39,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type()) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -59,12 +59,12 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity - match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags) + match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType) if err == filter.ErrExcluded { return false, filter.ErrExcluded diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61cb49ff1..1e84def62 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,3 +73,21 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + default: + return "" + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..36f66503e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -29,6 +29,10 @@ type Options struct { Severities severity.Severities // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities + // Protocols contains the protocols to be allowed executed + Protocols goflags.NormalizedStringSlice + // ExcludeProtocols contains protocols to not be executed + ExcludeProtocols goflags.NormalizedStringSlice // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From f0580298413c73e1647fb5c3e45610e95f799242 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 16:29:00 +0530 Subject: [PATCH 041/196] Misc --- v2/cmd/integration-test/http.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index b2ecc816b..c8bc6acc1 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -41,8 +41,6 @@ type httpInteractshRequest struct{} func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - value := r.Header.Get("url") if value != "" { if resp, _ := http.DefaultClient.Get(value); resp != nil { @@ -53,7 +51,7 @@ func (h *httpInteractshRequest) Execute(filePath string) error { ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } From 1dce8af04532f4fb53081441db7e3a65256bc9eb Mon Sep 17 00:00:00 2001 From: Jop Zitman Date: Tue, 26 Oct 2021 15:34:33 +0200 Subject: [PATCH 042/196] Linting issues --- v2/cmd/integration-test/loader.go | 10 ++++++++-- v2/pkg/catalog/loader/remote_loader.go | 4 ---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 13aa3f05e..8c44bc20a 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -35,7 +35,10 @@ func (h *remoteTemplateList) Execute(templateList string) error { if err != nil { w.WriteHeader(500) } - w.Write(file) + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } })) ts := httptest.NewServer(router) defer ts.Close() @@ -68,7 +71,10 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { if err != nil { w.WriteHeader(500) } - w.Write(file) + _, err = w.Write(file) + if err != nil { + w.WriteHeader(500) + } })) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index ec47ed51b..d8f71eda1 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -70,10 +70,6 @@ func getRemoteContent(URL string, w chan<- RemoteContentError, contentType Conte return } - if err != nil { - - } - scanner := bufio.NewScanner(response.Body) var templateList []string for scanner.Scan() { From a0318ffc8fce43a7524006a7bd1db7d08934b517 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 20:36:44 +0530 Subject: [PATCH 043/196] Started refactor of template compilation + protocol building --- v2/pkg/engine/engine.go | 9 +++ v2/pkg/engine/engine_test.go | 1 + v2/pkg/templates/compile.go | 145 +++++++++++++++++++++-------------- 3 files changed, 98 insertions(+), 57 deletions(-) create mode 100644 v2/pkg/engine/engine.go create mode 100644 v2/pkg/engine/engine_test.go diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go new file mode 100644 index 000000000..2bdb4a969 --- /dev/null +++ b/v2/pkg/engine/engine.go @@ -0,0 +1,9 @@ +package engine + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. This was something which was +// missing from the previous versions of nuclei. +type Engine struct { +} diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go new file mode 100644 index 000000000..00a22ef62 --- /dev/null +++ b/v2/pkg/engine/engine_test.go @@ -0,0 +1 @@ +package engine diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bf4e79b18..31446c9fc 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "reflect" "strings" "github.com/pkg/errors" @@ -69,11 +70,6 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplateInfo = template.Info options.TemplatePath = filePath - // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { - return nil, fmt.Errorf("no requests defined for %s", template.ID) - } - // Compile the workflow request if len(template.Workflows) > 0 { compiled := &template.Workflow @@ -83,56 +79,10 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute template.CompiledWorkflow.Options = &options } - // Compile the requests found - requests := []protocols.Request{} - if len(template.RequestsDNS) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsDNS { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) + if err := template.compileProtocolRequests(options); err != nil { + return nil, err } - if len(template.RequestsHTTP) > 0 { - if options.Options.OfflineHTTP { - operatorsList := []*operators.Operators{} - mainLoop: - for _, req := range template.RequestsHTTP { - for _, path := range req.Path { - if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { - break mainLoop - } - } - operatorsList = append(operatorsList, &req.Operators) - } - if len(operatorsList) > 0 { - options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) - } - } else { - for _, req := range template.RequestsHTTP { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - } - if len(template.RequestsFile) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsFile { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsNetwork) > 0 && !options.Options.OfflineHTTP { - for _, req := range template.RequestsNetwork { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } - if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { - for _, req := range template.RequestsHeadless { - requests = append(requests, req) - } - template.Executer = executer.NewExecuter(requests, &options) - } if template.Executer != nil { if err := template.Executer.Compile(); err != nil { return nil, errors.Wrap(err, "could not compile request") @@ -151,14 +101,95 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } // parseSelfContainedRequests parses the self contained template requests. -func (t *Template) parseSelfContainedRequests() { - if !t.SelfContained { +func (template *Template) parseSelfContainedRequests() { + if !template.SelfContained { return } - for _, request := range t.RequestsHTTP { + for _, request := range template.RequestsHTTP { request.SelfContained = true } - for _, request := range t.RequestsNetwork { + for _, request := range template.RequestsNetwork { request.SelfContained = true } } + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + len(template.RequestsHTTP) + + len(template.RequestsFile) + + len(template.RequestsNetwork) + + len(template.RequestsHeadless) + + len(template.Workflows) +} + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + templateRequests := template.Requests() + + if templateRequests == 0 { + return fmt.Errorf("no requests defined for %s", template.ID) + } + + if template.Options.Options.OfflineHTTP { + template.compileOfflineHTTPRequest(options) + return nil + } + + var requests []protocols.Request + switch { + case len(template.RequestsDNS) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + + case len(template.RequestsFile) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + + case len(template.RequestsNetwork) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + + case len(template.RequestsHTTP) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + + case len(template.RequestsHeadless) > 0 && options.Options.Headless: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} + +func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { + switch reflect.TypeOf(requests).Kind() { + case reflect.Slice: + s := reflect.ValueOf(requests) + + requestSlice := make([]protocols.Request, s.Len()) + for i := 0; i < s.Len(); i++ { + value := s.Index(i) + valueInterface := value.Interface() + requestSlice[i] = valueInterface.(protocols.Request) + } + return requestSlice + } + return nil +} + +// compileOfflineHTTPRequest iterates all requests if offline http mode is +// specified and collects all matchers for all the base request templates +// (those with URL {{BaseURL}} and it's slash variation.) +func (template *Template) compileOfflineHTTPRequest(options protocols.ExecuterOptions) { + operatorsList := []*operators.Operators{} + +mainLoop: + for _, req := range template.RequestsHTTP { + for _, path := range req.Path { + if !(strings.EqualFold(path, "{{BaseURL}}") || strings.EqualFold(path, "{{BaseURL}}/")) { + break mainLoop + } + } + operatorsList = append(operatorsList, &req.Operators) + } + if len(operatorsList) > 0 { + options.Operators = operatorsList + template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + } +} From 2a84b9eb4443b9a826ff4d692d254eed2e5ea52d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 26 Oct 2021 20:40:02 +0530 Subject: [PATCH 044/196] misc --- v2/pkg/templates/compile.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 31446c9fc..8c48308c5 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -139,19 +139,19 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti var requests []protocols.Request switch { case len(template.RequestsDNS) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsDNS) case len(template.RequestsFile) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsFile) case len(template.RequestsNetwork) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsNetwork) case len(template.RequestsHTTP) > 0: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsHTTP) case len(template.RequestsHeadless) > 0 && options.Options.Headless: - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + requests = template.convertRequestToProtocolsRequest(template.RequestsHeadless) } template.Executer = executer.NewExecuter(requests, &options) return nil From 97645dde520a0191cc87012e496720903a79bc43 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 15:53:04 +0530 Subject: [PATCH 045/196] Added new workpool package + Misc refactor --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/options.go | 7 -- v2/internal/runner/runner.go | 145 +++++------------------------- v2/pkg/engine/engine.go | 7 ++ v2/pkg/engine/inputs/hmap/hmap.go | 120 +++++++++++++++++++++++++ v2/pkg/engine/workpool.go | 37 ++++++++ v2/pkg/templates/compile.go | 5 +- v2/pkg/templates/templates.go | 30 +++++++ v2/pkg/types/types.go | 4 + 9 files changed, 228 insertions(+), 129 deletions(-) create mode 100644 v2/pkg/engine/inputs/hmap/hmap.go create mode 100644 v2/pkg/engine/workpool.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..13602bfcd 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -110,6 +110,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), + flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "hc", 10, "maximum number of headless templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe63..557821745 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -47,13 +47,6 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Program exiting: %s\n", err) } - // Auto adjust rate limits when using headless mode if the user - // hasn't specified any custom limits. - if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 { - options.BulkSize = 2 - options.TemplateThreads = 2 - } - // Load the resolvers if user asked for them loadResolvers(options) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f806952..d4d7f40c8 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,14 +16,12 @@ import ( "go.uber.org/ratelimit" "gopkg.in/yaml.v2" - "github.com/projectdiscovery/filekv" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -46,22 +44,20 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - hostMap *hybrid.HybridMap - hostMapStream *filekv.FileDB - output output.Writer - interactsh *interactsh.Client - inputCount int64 - templatesConfig *config.Config - options *types.Options - projectFile *projectfile.ProjectFile - catalog *catalog.Catalog - progress progress.Progress - colorizer aurora.Aurora - issuesClient *reporting.Client - addColor func(severity.Severity) string - browser *engine.Browser - ratelimiter ratelimit.Limiter - hostErrors *hosterrorscache.Cache + output output.Writer + interactsh *interactsh.Client + templatesConfig *config.Config + options *types.Options + projectFile *projectfile.ProjectFile + catalog *catalog.Catalog + progress progress.Progress + colorizer aurora.Aurora + issuesClient *reporting.Client + addColor func(severity.Severity) string + hmapInputProvider *hmap.Input + browser *engine.Browser + ratelimiter ratelimit.Limiter + hostErrors *hosterrorscache.Cache } // New creates a new client for running enumeration process. @@ -116,103 +112,13 @@ func New(options *types.Options) (*Runner, error) { if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates { os.Exit(0) } - hm, err := hybrid.New(hybrid.DefaultDiskOptions) + + // Initialize the input source + hmapInput, err := hmap.New(options) if err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } - runner.hostMap = hm - - if options.Stream { - fkvOptions := filekv.DefaultOptions - if tmpFileName, err := fileutil.GetTempFileName(); err != nil { - return nil, errors.Wrap(err, "could not create temporary input file") - } else { - fkvOptions.Path = tmpFileName - } - fkv, err := filekv.Open(fkvOptions) - if err != nil { - return nil, errors.Wrap(err, "could not create temporary unsorted input file") - } - runner.hostMapStream = fkv - } - - runner.inputCount = 0 - dupeCount := 0 - - // Handle multiple targets - if len(options.Targets) != 0 { - for _, target := range options.Targets { - url := strings.TrimSpace(target) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle stdin - if options.Stdin { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - } - - // Handle target file - if options.TargetsFilePath != "" { - input, inputErr := os.Open(options.TargetsFilePath) - if inputErr != nil { - return nil, errors.Wrap(inputErr, "could not open targets file") - } - scanner := bufio.NewScanner(input) - for scanner.Scan() { - url := strings.TrimSpace(scanner.Text()) - if url == "" { - continue - } - if _, ok := runner.hostMap.Get(url); ok { - dupeCount++ - continue - } - runner.inputCount++ - // nolint:errcheck // ignoring error - runner.hostMap.Set(url, nil) - if options.Stream { - _ = runner.hostMapStream.Set([]byte(url), nil) - } - } - input.Close() - } - - if dupeCount > 0 { - gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount) + return nil, errors.Wrap(err, "could not create input provider") } + runner.hmapInputProvider = hmapInput // Create the output file if asked outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile) @@ -312,13 +218,10 @@ func (r *Runner) Close() { if r.output != nil { r.output.Close() } - r.hostMap.Close() if r.projectFile != nil { r.projectFile.Close() } - if r.options.Stream { - r.hostMapStream.Close() - } + r.hmapInputProvider.Close() protocolinit.Close() } @@ -456,7 +359,7 @@ func (r *Runner) RunEnumeration() error { if len(template.Workflows) > 0 { continue } - unclusteredRequests += int64(template.TotalRequests) * r.inputCount + unclusteredRequests += int64(template.TotalRequests) * r.hmapInputProvider.Count() } if r.options.VerboseVerbose { @@ -509,7 +412,7 @@ func (r *Runner) RunEnumeration() error { if len(t.Workflows) > 0 { continue } - totalRequests += int64(t.TotalRequests) * r.inputCount + totalRequests += int64(t.TotalRequests) * r.hmapInputProvider.Count() } if totalRequests < unclusteredRequests { gologger.Info().Msgf("Templates clustered: %d (Reduced %d HTTP Requests)", clusterCount, unclusteredRequests-totalRequests) @@ -531,7 +434,7 @@ func (r *Runner) RunEnumeration() error { wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) // tracks global progress and captures stdout/stderr until p.Wait finishes - r.progress.Init(r.inputCount, templateCount, totalRequests) + r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) for _, t := range finalTemplates { wgtemplates.Add() diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go index 2bdb4a969..978e845ca 100644 --- a/v2/pkg/engine/engine.go +++ b/v2/pkg/engine/engine.go @@ -7,3 +7,10 @@ package engine // missing from the previous versions of nuclei. type Engine struct { } + +// InputProvider is an input provider interface for the nuclei execution +// engine. +// +// An example InputProvider is provided in form of hmap input provider. +type InputProvider interface { +} diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/engine/inputs/hmap/hmap.go new file mode 100644 index 000000000..2910d0451 --- /dev/null +++ b/v2/pkg/engine/inputs/hmap/hmap.go @@ -0,0 +1,120 @@ +// Package hmap implements a hybrid hmap/filekv backed input provider +// for nuclei that can either stream or store results using different kv stores. +package hmap + +import ( + "bufio" + "io" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Input is a hmap/filekv backed nuclei Input provider +type Input struct { + inputCount int64 + dupeCount int64 + hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB +} + +// New creates a new hmap backed nuclei Input Provider +// and initializes it based on the passed options Model. +func New(options *types.Options) (*Input, error) { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } + + input := &Input{hostMap: hm} + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + input.hostMapStream = fkv + } + if initErr := input.initializeInputSources(options); initErr != nil { + return nil, initErr + } + if input.dupeCount > 0 { + gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", input.dupeCount) + } + return input, nil +} + +// Close closes the input provider +func (i *Input) Close() { + i.hostMap.Close() + if i.hostMapStream != nil { + i.hostMapStream.Close() + } +} + +// initializeInputSources initializes the input sources for hmap input +func (i *Input) initializeInputSources(options *types.Options) error { + // Handle targets flags + for _, target := range options.Targets { + i.normalizeStoreInputValue(target) + } + + // Handle stdin + if options.Stdin { + i.scanInputFromReader(os.Stdin) + } + + // Handle target file + if options.TargetsFilePath != "" { + input, inputErr := os.Open(options.TargetsFilePath) + if inputErr != nil { + return errors.Wrap(inputErr, "could not open targets file") + } + i.scanInputFromReader(input) + input.Close() + } + return nil +} + +// scanInputFromReader scans a line of input from reader and passes it for storage +func (i *Input) scanInputFromReader(reader io.Reader) { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + i.normalizeStoreInputValue(scanner.Text()) + } +} + +// normalizeStoreInputValue normalizes and stores passed input values +func (i *Input) normalizeStoreInputValue(value string) { + url := strings.TrimSpace(value) + if url == "" { + return + } + + if _, ok := i.hostMap.Get(url); ok { + i.dupeCount++ + return + } + + i.inputCount++ + _ = i.hostMap.Set(url, nil) + if i.hostMapStream != nil { + _ = i.hostMapStream.Set([]byte(url), nil) + } +} + +// Count returns the input count +func (i *Input) Count() int64 { + return i.inputCount +} diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/engine/workpool.go new file mode 100644 index 000000000..04db44715 --- /dev/null +++ b/v2/pkg/engine/workpool.go @@ -0,0 +1,37 @@ +package engine + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/templates" +) + +// WorkPool implements an execution pool for executing different +// types of task with different concurreny requirements. +// +// It also allows Configuration of such requirements. This is used +// for per-module like separate headless concurrency etc. +type WorkPool struct { + config WorkPoolConfig +} + +// WorkPoolConfig is the configuration for workpool +type WorkPoolConfig struct { + // InputConcurrency is the concurrency for inputs values. + InputConcurrency int + // TypeConcurrency is the concurrency for the request type templates. + TypeConcurrency int + // HeadlessInputConcurrency is the concurrency for headless inputs values. + HeadlessInputConcurrency int + // TypeConcurrency is the concurrency for the headless request type templates. + HeadlessTypeConcurrency int +} + +// New returns a new WorkPool instance +func New(config WorkPoolConfig) *WorkPool { + return &WorkPool{config: config} +} + +// TODO: port these invocations of waitgroups and input logic into a generic +// workpool type functionality. +func (w *WorkPool) Execute(templates []*templates.Template) { + +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 8c48308c5..22c2d764f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -131,7 +131,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti return fmt.Errorf("no requests defined for %s", template.ID) } - if template.Options.Options.OfflineHTTP { + if options.Options.OfflineHTTP { template.compileOfflineHTTPRequest(options) return nil } @@ -157,6 +157,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti return nil } +// convertRequestToProtocolsRequest is a convenience wrapper to convert +// arbitrary interfaces which are slices of requests from the template to a +// slice of protocols.Request interface items. func (template *Template) convertRequestToProtocolsRequest(requests interface{}) []protocols.Request { switch reflect.TypeOf(requests).Kind() { case reflect.Slice: diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61cb49ff1..7e0e42a37 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,3 +73,33 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// TemplateTypes is a list of accepted template types +var TemplateTypes = []string{ + "dns", + "file", + "http", + "headless", + "network", + "workflow", +} + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + case len(t.Workflows) > 0: + return "workflow" + default: + return "" + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..9f8f784b1 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -77,6 +77,10 @@ type Options struct { BulkSize int // TemplateThreads is the number of templates executed in parallel TemplateThreads int + // HeadlessBulkSize is the of targets analyzed in parallel for each headless template + HeadlessBulkSize int + // HeadlessTemplateThreads is the number of headless templates executed in parallel + HeadlessTemplateThreads int // Timeout is the seconds to wait for a response from the server. Timeout int // Retries is the number of times to retry the request From c16c93fe7ca698d3086b85fd86e0c1e8c99a222e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 16:50:36 +0530 Subject: [PATCH 046/196] refactor the modules to core --- v2/internal/runner/runner.go | 46 +++------------- v2/pkg/core/engine.go | 53 +++++++++++++++++++ v2/pkg/core/engine_test.go | 1 + .../processor.go => pkg/core/execute.go} | 46 +++++++++++++++- v2/pkg/{engine => core}/inputs/hmap/hmap.go | 0 .../execute.go => core/workflow_execute.go} | 12 ++--- .../workflow_execute_test.go} | 39 +++++++------- v2/pkg/{engine => core}/workpool.go | 6 +-- v2/pkg/engine/engine.go | 16 ------ v2/pkg/engine/engine_test.go | 1 - v2/pkg/protocols/protocols.go | 6 +++ v2/pkg/templates/workflows.go | 14 +---- 12 files changed, 139 insertions(+), 101 deletions(-) create mode 100644 v2/pkg/core/engine.go create mode 100644 v2/pkg/core/engine_test.go rename v2/{internal/runner/processor.go => pkg/core/execute.go} (61%) rename v2/pkg/{engine => core}/inputs/hmap/hmap.go (100%) rename v2/pkg/{workflows/execute.go => core/workflow_execute.go} (95%) rename v2/pkg/{workflows/execute_test.go => core/workflow_execute_test.go} (77%) rename v2/pkg/{engine => core}/workpool.go (90%) delete mode 100644 v2/pkg/engine/engine.go delete mode 100644 v2/pkg/engine/engine_test.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index d4d7f40c8..100e364f0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -2,7 +2,6 @@ package runner import ( "bufio" - "fmt" "os" "path/filepath" "strings" @@ -11,7 +10,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" - "github.com/rs/xid" "go.uber.org/atomic" "go.uber.org/ratelimit" "gopkg.in/yaml.v2" @@ -21,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -28,7 +27,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" @@ -247,6 +245,9 @@ func (r *Runner) RunEnumeration() error { cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) } r.hostErrors = cache + + // Create the executer options which will be used throughout the execution + // stage by the nuclei engine modules. executerOpts := protocols.ExecuterOptions{ Output: r.output, Options: r.options, @@ -259,12 +260,13 @@ func (r *Runner) RunEnumeration() error { Browser: r.browser, HostErrorsCache: cache, } + engine := core.New(r.options) + engine.SetExecuterOptions(executerOpts) workflowLoader, err := parsers.NewLoader(&executerOpts) if err != nil { return errors.Wrap(err, "Could not create loader.") } - executerOpts.WorkflowLoader = workflowLoader loaderConfig := loader.Config{ @@ -370,42 +372,6 @@ func (r *Runner) RunEnumeration() error { r.logAvailableTemplate(template.Path) } } - templatesMap := make(map[string]*templates.Template) - for _, v := range store.Templates() { - templatesMap[v.Path] = v - } - originalTemplatesCount := len(store.Templates()) - clusterCount := 0 - clusters := clusterer.Cluster(templatesMap) - for _, cluster := range clusters { - if len(cluster) > 1 && !r.options.OfflineHTTP { - executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - RateLimiter: r.ratelimiter, - IssuesClient: r.issuesClient, - Browser: r.browser, - ProjectFile: r.projectFile, - Interactsh: r.interactsh, - HostErrorsCache: cache, - } - clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - - finalTemplates = append(finalTemplates, &templates.Template{ - ID: clusterID, - RequestsHTTP: cluster[0].RequestsHTTP, - Executer: clusterer.NewExecuter(cluster, &executerOpts), - TotalRequests: len(cluster[0].RequestsHTTP), - }) - clusterCount += len(cluster) - } else { - finalTemplates = append(finalTemplates, cluster...) - } - } - - finalTemplates = append(finalTemplates, store.Workflows()...) var totalRequests int64 for _, t := range finalTemplates { diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go new file mode 100644 index 000000000..d7fdc6bbc --- /dev/null +++ b/v2/pkg/core/engine.go @@ -0,0 +1,53 @@ +package core + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Engine is an engine for running Nuclei Templates/Workflows. +// +// The engine contains multiple thread pools which allow using different +// concurrency values per protocol executed. +// +// The engine does most of the heavy lifting of execution, from clustering +// templates to leading to the final execution by the workpool, it is +// handled by the engine. +type Engine struct { + workPool *WorkPool + options *types.Options + executerOpts protocols.ExecuterOptions +} + +// InputProvider is an input provider interface for the nuclei execution +// engine. +// +// An example InputProvider is provided in form of hmap input provider. +type InputProvider interface { +} + +// New returns a new Engine instance +func New(options *types.Options) *Engine { + workPool := NewWorkPool(WorkPoolConfig{ + InputConcurrency: options.BulkSize, + TypeConcurrency: options.TemplateThreads, + HeadlessInputConcurrency: options.HeadlessBulkSize, + HeadlessTypeConcurrency: options.HeadlessTemplateThreads, + }) + engine := &Engine{ + options: options, + workPool: workPool, + } + return engine +} + +// SetExecuterOptions sets the executer options for the engine. This is required +// before using the engine to perform any execution. +func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) { + e.executerOpts = options +} + +// ExecuterOptions returns protocols.ExecuterOptions for nuclei engine. +func (e *Engine) ExecuterOptions() protocols.ExecuterOptions { + return e.executerOpts +} diff --git a/v2/pkg/core/engine_test.go b/v2/pkg/core/engine_test.go new file mode 100644 index 000000000..9a8bc9592 --- /dev/null +++ b/v2/pkg/core/engine_test.go @@ -0,0 +1 @@ +package core diff --git a/v2/internal/runner/processor.go b/v2/pkg/core/execute.go similarity index 61% rename from v2/internal/runner/processor.go rename to v2/pkg/core/execute.go index 6b14b5f91..65c31558d 100644 --- a/v2/internal/runner/processor.go +++ b/v2/pkg/core/execute.go @@ -1,5 +1,48 @@ -package runner +package core +import ( + "fmt" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/rs/xid" +) + +// clusterTemplates performs identical http requests clustering for a list of templates +func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { + if e.options.OfflineHTTP { + return templatesList, 0 + } + + templatesMap := make(map[string]*templates.Template) + for _, v := range templatesList { + templatesMap[v.Path] = v + } + clusterCount := 0 + + finalTemplatesList := make([]*templates.Template, 0, len(templatesList)) + clusters := clusterer.Cluster(templatesMap) + for _, cluster := range clusters { + if len(cluster) > 1 { + executerOpts := e.ExecuterOptions() + + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + finalTemplatesList = append(finalTemplatesList, &templates.Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: clusterer.NewExecuter(cluster, &executerOpts), + TotalRequests: len(cluster[0].RequestsHTTP), + }) + clusterCount += len(cluster) + } else { + finalTemplatesList = append(finalTemplatesList, cluster...) + } + } + return finalTemplatesList, clusterCount +} + +/* import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -79,3 +122,4 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { wg.Wait() return results.Load() } +*/ diff --git a/v2/pkg/engine/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hmap/hmap.go similarity index 100% rename from v2/pkg/engine/inputs/hmap/hmap.go rename to v2/pkg/core/inputs/hmap/hmap.go diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/core/workflow_execute.go similarity index 95% rename from v2/pkg/workflows/execute.go rename to v2/pkg/core/workflow_execute.go index c0710c6ad..59a5ade55 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,12 +1,6 @@ -package workflows - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" -) +package core +/* // RunWorkflow runs a workflow on an input and returns true or false func (w *Workflow) RunWorkflow(input string) bool { results := &atomic.Bool{} @@ -123,4 +117,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } return mainErr -} +}*/ diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/core/workflow_execute_test.go similarity index 77% rename from v2/pkg/workflows/execute_test.go rename to v2/pkg/core/workflow_execute_test.go index 6d9ab6a09..62cd40840 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -1,5 +1,6 @@ -package workflows +package core +/* import ( "testing" @@ -10,13 +11,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) func TestWorkflowsSimple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} @@ -29,13 +31,13 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, - {Executers: []*ProtocolExecuterPair{{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -53,14 +55,14 @@ func TestWorkflowsSubtemplates(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -78,12 +80,12 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -101,8 +103,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -111,7 +113,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "tomcat", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -129,8 +131,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ - {Executers: []*ProtocolExecuterPair{{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ + {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ @@ -139,7 +141,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, - }, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{ + }, Matchers: []*workflows.Matcher{{Name: "apache", Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -187,3 +189,4 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu } return nil } +*/ diff --git a/v2/pkg/engine/workpool.go b/v2/pkg/core/workpool.go similarity index 90% rename from v2/pkg/engine/workpool.go rename to v2/pkg/core/workpool.go index 04db44715..f307b205d 100644 --- a/v2/pkg/engine/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,4 +1,4 @@ -package engine +package core import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -25,8 +25,8 @@ type WorkPoolConfig struct { HeadlessTypeConcurrency int } -// New returns a new WorkPool instance -func New(config WorkPoolConfig) *WorkPool { +// NewWorkPool returns a new WorkPool instance +func NewWorkPool(config WorkPoolConfig) *WorkPool { return &WorkPool{config: config} } diff --git a/v2/pkg/engine/engine.go b/v2/pkg/engine/engine.go deleted file mode 100644 index 978e845ca..000000000 --- a/v2/pkg/engine/engine.go +++ /dev/null @@ -1,16 +0,0 @@ -package engine - -// Engine is an engine for running Nuclei Templates/Workflows. -// -// The engine contains multiple thread pools which allow using different -// concurrency values per protocol executed. This was something which was -// missing from the previous versions of nuclei. -type Engine struct { -} - -// InputProvider is an input provider interface for the nuclei execution -// engine. -// -// An example InputProvider is provided in form of hmap input provider. -type InputProvider interface { -} diff --git a/v2/pkg/engine/engine_test.go b/v2/pkg/engine/engine_test.go deleted file mode 100644 index 00a22ef62..000000000 --- a/v2/pkg/engine/engine_test.go +++ /dev/null @@ -1 +0,0 @@ -package engine diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index d77ba147e..83a76f7cf 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -64,6 +64,12 @@ type ExecuterOptions struct { WorkflowLoader model.WorkflowLoader } +// Copy returns a copy of the executeroptions structure +func (e ExecuterOptions) Copy() ExecuterOptions { + copy := e + return copy +} + // Request is an interface implemented any protocol based request generator. type Request interface { // Compile compiles the request generators preparing any requests possible. diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 9d510f2e2..246c4bf8e 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -62,19 +62,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr return nil } for _, path := range paths { - opts := protocols.ExecuterOptions{ - Output: options.Output, - Options: options.Options, - Progress: options.Progress, - Catalog: options.Catalog, - Browser: options.Browser, - RateLimiter: options.RateLimiter, - IssuesClient: options.IssuesClient, - Interactsh: options.Interactsh, - ProjectFile: options.ProjectFile, - HostErrorsCache: options.HostErrorsCache, - } - template, err := Parse(path, preprocessor, opts) + template, err := Parse(path, preprocessor, options.Copy()) if err != nil { gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err) continue From df78ea72c5c266fecc4c01e97a700025a921ed42 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 27 Oct 2021 18:41:39 +0530 Subject: [PATCH 047/196] misc --- v2/pkg/core/execute.go | 24 ++++++++++++++++++++++++ v2/pkg/core/workpool.go | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 65c31558d..112be4e2c 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -8,6 +8,30 @@ import ( "github.com/rs/xid" ) +// Execute takes a list of templates/workflows that have been compiled +// and executes them based on provided concurrency options. +// +// All the execution logic for the templates/workflows happens in this part +// of the engine. +func (e *Engine) Execute(templates []*templates.Template) { + finalTemplates, clusterCount := e.clusterTemplates(templates) + + for _, template := range finalTemplates { + templateType := template.Type() + + switch { + case template.SelfContained: + // Self Contained requests are executed here + + case templateType == "workflow": + // Workflows requests are executed here + + default: + // All other request types are executed here + } + } +} + // clusterTemplates performs identical http requests clustering for a list of templates func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { if e.options.OfflineHTTP { diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index f307b205d..ef50f70fc 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -30,8 +30,6 @@ func NewWorkPool(config WorkPoolConfig) *WorkPool { return &WorkPool{config: config} } -// TODO: port these invocations of waitgroups and input logic into a generic -// workpool type functionality. func (w *WorkPool) Execute(templates []*templates.Template) { } From a539184ffd9701368f93be55197a22a99639d79e Mon Sep 17 00:00:00 2001 From: kchason Date: Wed, 27 Oct 2021 12:11:42 -0400 Subject: [PATCH 048/196] Switch logic to a shared package --- .../protocols/headless/engine/http_client.go | 26 ++------------ .../http/httpclientpool/clientpool.go | 26 ++------------ v2/pkg/protocols/utils/utils.go | 34 +++++++++++++++++++ 3 files changed, 40 insertions(+), 46 deletions(-) create mode 100644 v2/pkg/protocols/utils/utils.go diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index d2f48c596..0edf6bf6e 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -2,9 +2,7 @@ package engine import ( "crypto/tls" - "crypto/x509" - "io/ioutil" - "log" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net/http" "net/url" "time" @@ -23,26 +21,8 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, } - // Build the TLS config with the client certificate if it has been configured with the appropriate options. - // Only one of the options needs to be checked since the validation checks in main.go ensure that all three - // files are set if any of the client certification configuration options are. - if len(options.ClientCertFile) > 0 { - // Load the client certificate using the PEM encoded client certificate and the private key file - cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) - if err != nil { - log.Fatal(err) - } - tlsConfig.Certificates = []tls.Certificate{cert} - - // Load the certificate authority PEM certificate into the TLS configuration - caCert, err := ioutil.ReadFile(options.ClientCAFile) - if err != nil { - log.Fatal(err) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - tlsConfig.RootCAs = caCertPool - } + // Add the client certificate authentication to the request if it's configured + tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) transport := &http.Transport{ DialContext: dialer.Dial, diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 3b0ce9b48..2b7dc0245 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -3,10 +3,8 @@ package httpclientpool import ( "context" "crypto/tls" - "crypto/x509" "fmt" - "io/ioutil" - "log" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" @@ -170,26 +168,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl InsecureSkipVerify: true, } - // Build the TLS config with the client certificate if it has been configured with the appropriate options. - // Only one of the options needs to be checked since the validation checks in main.go ensure that all three - // files are set if any of the client certification configuration options are. - if len(options.ClientCertFile) > 0 { - // Load the client certificate using the PEM encoded client certificate and the private key file - cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) - if err != nil { - log.Fatal(err) - } - tlsConfig.Certificates = []tls.Certificate{cert} - - // Load the certificate authority PEM certificate into the TLS configuration - caCert, err := ioutil.ReadFile(options.ClientCAFile) - if err != nil { - log.Fatal(err) - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - tlsConfig.RootCAs = caCertPool - } + // Add the client certificate authentication to the request if it's configured + tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) transport := &http.Transport{ DialContext: Dialer.Dial, diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go new file mode 100644 index 000000000..8d43fd478 --- /dev/null +++ b/v2/pkg/protocols/utils/utils.go @@ -0,0 +1,34 @@ +package utils + +import ( + "crypto/tls" + "crypto/x509" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "io/ioutil" + "log" +) + +// AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it +func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config { + // Build the TLS config with the client certificate if it has been configured with the appropriate options. + // Only one of the options needs to be checked since the validation checks in main.go ensure that all three + // files are set if any of the client certification configuration options are. + if len(options.ClientCertFile) > 0 { + // Load the client certificate using the PEM encoded client certificate and the private key file + cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) + if err != nil { + log.Fatal(err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load the certificate authority PEM certificate into the TLS configuration + caCert, err := ioutil.ReadFile(options.ClientCAFile) + if err != nil { + log.Fatal(err) + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + tlsConfig.RootCAs = caCertPool + } + return tlsConfig +} From d124dbacc7cf3b3ce4328c391a908cedd68703d6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 17:20:07 +0530 Subject: [PATCH 049/196] Moved all important execution stuff to engine --- v2/internal/runner/runner.go | 43 ++---- v2/pkg/core/engine.go | 4 + v2/pkg/core/execute.go | 162 +++++++++----------- v2/pkg/core/inputs/{hmap => hybrid}/hmap.go | 17 +- v2/pkg/core/workflow_execute.go | 29 ++-- v2/pkg/core/workflow_execute_test.go | 30 ++-- v2/pkg/core/workpool.go | 38 ++++- v2/pkg/protocols/protocols.go | 2 + 8 files changed, 171 insertions(+), 154 deletions(-) rename v2/pkg/core/inputs/{hmap => hybrid}/hmap.go (88%) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 100e364f0..84f2837a8 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -9,8 +9,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" "go.uber.org/ratelimit" "gopkg.in/yaml.v2" @@ -20,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" - "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hmap" + "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" @@ -34,7 +32,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -52,7 +49,7 @@ type Runner struct { colorizer aurora.Aurora issuesClient *reporting.Client addColor func(severity.Severity) string - hmapInputProvider *hmap.Input + hmapInputProvider *hybrid.Input browser *engine.Browser ratelimiter ratelimit.Limiter hostErrors *hosterrorscache.Cache @@ -112,7 +109,7 @@ func New(options *types.Options) (*Runner, error) { } // Initialize the input source - hmapInput, err := hmap.New(options) + hmapInput, err := hybrid.New(options) if err != nil { return nil, errors.Wrap(err, "could not create input provider") } @@ -259,6 +256,7 @@ func (r *Runner) RunEnumeration() error { ProjectFile: r.projectFile, Browser: r.browser, HostErrorsCache: cache, + Colorizer: r.colorizer, } engine := core.New(r.options) engine.SetExecuterOptions(executerOpts) @@ -351,9 +349,6 @@ func (r *Runner) RunEnumeration() error { gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) } - // pre-parse all the templates, apply filters - finalTemplates := []*templates.Template{} - var unclusteredRequests int64 for _, template := range store.Templates() { // workflows will dynamically adjust the totals while running, as @@ -373,6 +368,12 @@ func (r *Runner) RunEnumeration() error { } } + // Cluster the templates first because we want info on how many + // templates did we cluster for showing to user in CLI + originalTemplatesCount := len(store.Templates()) + finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates()) + finalTemplates = append(finalTemplates, store.Workflows()...) + var totalRequests int64 for _, t := range finalTemplates { if len(t.Workflows) > 0 { @@ -391,32 +392,10 @@ func (r *Runner) RunEnumeration() error { return errors.New("no valid templates were found") } - /* - TODO does it make sense to run the logic below if there are no targets specified? - Can we safely assume the user is just experimenting with the template/workflow filters before running them? - */ - - results := &atomic.Bool{} - wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) - // tracks global progress and captures stdout/stderr until p.Wait finishes r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests) - for _, t := range finalTemplates { - wgtemplates.Add() - go func(template *templates.Template) { - defer wgtemplates.Done() - - if template.SelfContained { - results.CAS(false, r.processSelfContainedTemplates(template)) - } else if len(template.Workflows) > 0 { - results.CAS(false, r.processWorkflowWithList(template)) - } else { - results.CAS(false, r.processTemplateWithList(template)) - } - }(t) - } - wgtemplates.Wait() + results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true) if r.interactsh != nil { matched := r.interactsh.Close() diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index d7fdc6bbc..455546244 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -24,6 +24,10 @@ type Engine struct { // // An example InputProvider is provided in form of hmap input provider. type InputProvider interface { + // Count returns the number of items for input provider + Count() int64 + // Scan calls a callback function till the input provider is exhausted + Scan(callback func(value string)) } // New returns a new Engine instance diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 112be4e2c..3593123a2 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -3,9 +3,12 @@ package core import ( "fmt" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" "github.com/rs/xid" + "go.uber.org/atomic" ) // Execute takes a list of templates/workflows that have been compiled @@ -13,27 +16,86 @@ import ( // // All the execution logic for the templates/workflows happens in this part // of the engine. -func (e *Engine) Execute(templates []*templates.Template) { - finalTemplates, clusterCount := e.clusterTemplates(templates) +func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool { + return e.ExecuteWithOpts(templates, input, false) +} +// ExecuteWithOpts is execute with the full options +func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool { + var finalTemplates []*templates.Template + if !noCluster { + finalTemplates, _ = e.ClusterTemplates(templatesList) + } else { + finalTemplates = templatesList + } + + results := &atomic.Bool{} for _, template := range finalTemplates { templateType := template.Type() + var wg *sizedwaitgroup.SizedWaitGroup + if templateType == "headless" { + wg = e.workPool.Headless + } else { + wg = e.workPool.Default + } + + wg.Add() switch { case template.SelfContained: - // Self Contained requests are executed here - - case templateType == "workflow": - // Workflows requests are executed here - + // Self Contained requests are executed here separately + e.executeSelfContainedTemplateWithInput(template, results) default: // All other request types are executed here + e.executeModelWithInput(templateType, template, input, results) } } + e.workPool.Wait() + return results } -// clusterTemplates performs identical http requests clustering for a list of templates -func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { +// processSelfContainedTemplates execute a self-contained template. +func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) +} + +// executeModelWithInput executes a type of template with input +func (e *Engine) executeModelWithInput(templateType string, template *templates.Template, input InputProvider, results *atomic.Bool) { + wg := e.workPool.InputPool(templateType) + + input.Scan(func(scannedValue string) { + // Skip if the host has had errors + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { + return + } + + wg.Waitgroup.Add() + go func(value string) { + defer wg.Waitgroup.Done() + + var match bool + var err error + switch templateType { + case "workflow": + match = e.executeWorkflow(value, template.CompiledWorkflow) + default: + match, err = template.Executer.Execute(value) + } + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) + } + results.CAS(false, match) + }(scannedValue) + }) + wg.Waitgroup.Wait() +} + +// ClusterTemplates performs identical http requests clustering for a list of templates +func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { if e.options.OfflineHTTP { return templatesList, 0 } @@ -65,85 +127,3 @@ func (e *Engine) clusterTemplates(templatesList []*templates.Template) ([]*templ } return finalTemplatesList, clusterCount } - -/* -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" -) - -// processSelfContainedTemplates execute a self-contained template. -func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { - match, err := template.Executer.Execute("") - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - return match -} - -// processTemplateWithList execute a template against the list of user provided targets -func (r *Runner) processTemplateWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - - match, err := template.Executer.Execute(URL) - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } - results.CAS(false, match) - }(URL) - return nil - } - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} - -// processTemplateWithList process a template on the URL list -func (r *Runner) processWorkflowWithList(template *templates.Template) bool { - results := &atomic.Bool{} - wg := sizedwaitgroup.New(r.options.BulkSize) - - processItem := func(k, _ []byte) error { - URL := string(k) - - // Skip if the host has had errors - if r.hostErrors != nil && r.hostErrors.Check(URL) { - return nil - } - wg.Add() - go func(URL string) { - defer wg.Done() - match := template.CompiledWorkflow.RunWorkflow(URL) - results.CAS(false, match) - }(URL) - return nil - } - - if r.options.Stream { - _ = r.hostMapStream.Scan(processItem) - } else { - r.hostMap.Scan(processItem) - } - - wg.Wait() - return results.Load() -} -*/ diff --git a/v2/pkg/core/inputs/hmap/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go similarity index 88% rename from v2/pkg/core/inputs/hmap/hmap.go rename to v2/pkg/core/inputs/hybrid/hmap.go index 2910d0451..1eb4f5e42 100644 --- a/v2/pkg/core/inputs/hmap/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -1,6 +1,6 @@ -// Package hmap implements a hybrid hmap/filekv backed input provider +// Package hybrid implements a hybrid hmap/filekv backed input provider // for nuclei that can either stream or store results using different kv stores. -package hmap +package hybrid import ( "bufio" @@ -118,3 +118,16 @@ func (i *Input) normalizeStoreInputValue(value string) { func (i *Input) Count() int64 { return i.inputCount } + +// Scan calls an input provider till the callback is exhausted +func (i *Input) Scan(callback func(value string)) { + callbackFunc := func(k, _ []byte) error { + callback(string(k)) + return nil + } + if i.hostMapStream != nil { + _ = i.hostMapStream.Scan(callbackFunc) + } else { + i.hostMap.Scan(callbackFunc) + } +} diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 59a5ade55..8c255d7ad 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,15 +1,22 @@ package core -/* -// RunWorkflow runs a workflow on an input and returns true or false -func (w *Workflow) RunWorkflow(input string) bool { +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" +) + +// executeWorkflow runs a workflow on an input and returns true or false +func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { results := &atomic.Bool{} swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) for _, template := range w.Workflows { swg.Add() - func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, &swg); err != nil { + func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -21,7 +28,7 @@ func (w *Workflow) RunWorkflow(input string) bool { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error @@ -84,8 +91,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range matcher.Subtemplates { swg.Add() - go func(subtemplate *WorkflowTemplate) { - if err := w.runWorkflowStep(subtemplate, input, results, swg); err != nil { + go func(subtemplate *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err) } swg.Done() @@ -108,8 +115,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for _, subtemplate := range template.Subtemplates { swg.Add() - go func(template *WorkflowTemplate) { - if err := w.runWorkflowStep(template, input, results, swg); err != nil { + go func(template *workflows.WorkflowTemplate) { + if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } swg.Done() @@ -117,4 +124,4 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } return mainErr -}*/ +} diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index 62cd40840..a00ce6043 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -1,6 +1,5 @@ package core -/* import ( "testing" @@ -17,13 +16,14 @@ import ( func TestWorkflowsSimple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") } @@ -31,7 +31,7 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -44,7 +44,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { }}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -55,7 +56,7 @@ func TestWorkflowsSubtemplates(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -69,7 +70,8 @@ func TestWorkflowsSubtemplates(t *testing.T) { }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -80,7 +82,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input @@ -92,7 +94,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { }}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -103,7 +106,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { progressBar, _ := progress.NewStatsTicker(0, false, false, false, 0) var firstInput, secondInput string - workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*WorkflowTemplate{ + workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input @@ -120,7 +123,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -148,7 +152,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }}}}}}, }} - matched := workflow.RunWorkflow("https://test.com") + engine := &Engine{} + matched := engine.executeWorkflow("https://test.com", workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -189,4 +194,3 @@ func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.Outpu } return nil } -*/ diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index ef50f70fc..23b50d682 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,7 +1,7 @@ package core import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" ) // WorkPool implements an execution pool for executing different @@ -10,7 +10,9 @@ import ( // It also allows Configuration of such requirements. This is used // for per-module like separate headless concurrency etc. type WorkPool struct { - config WorkPoolConfig + Headless *sizedwaitgroup.SizedWaitGroup + Default *sizedwaitgroup.SizedWaitGroup + config WorkPoolConfig } // WorkPoolConfig is the configuration for workpool @@ -27,9 +29,35 @@ type WorkPoolConfig struct { // NewWorkPool returns a new WorkPool instance func NewWorkPool(config WorkPoolConfig) *WorkPool { - return &WorkPool{config: config} + headlessWg := sizedwaitgroup.New(config.HeadlessTypeConcurrency) + defaultWg := sizedwaitgroup.New(config.TypeConcurrency) + + return &WorkPool{ + config: config, + Headless: &headlessWg, + Default: &defaultWg, + } } -func (w *WorkPool) Execute(templates []*templates.Template) { - +// Wait waits for all the workpool waitgroups to finish +func (w *WorkPool) Wait() { + w.Default.Wait() + w.Headless.Wait() +} + +// InputWorkPool is a workpool per-input +type InputWorkPool struct { + Waitgroup *sizedwaitgroup.SizedWaitGroup +} + +// InputPool returns a workpool for an input type +func (w *WorkPool) InputPool(templateType string) *InputWorkPool { + var count int + if templateType == "headless" { + count = w.config.HeadlessInputConcurrency + } else { + count = w.config.InputConcurrency + } + swg := sizedwaitgroup.New(count) + return &InputWorkPool{Waitgroup: &swg} } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 83a76f7cf..d877328fe 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -3,6 +3,7 @@ package protocols import ( "go.uber.org/ratelimit" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -61,6 +62,7 @@ type ExecuterOptions struct { Operators []*operators.Operators // only used by offlinehttp module + Colorizer aurora.Aurora WorkflowLoader model.WorkflowLoader } From 0abc7202b1c4cd12da364100a2309ef8a9ca2ae1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 17:45:38 +0530 Subject: [PATCH 050/196] Misc fixes with goflags --- v2/go.mod | 73 +++++++++++++++++------------- v2/go.sum | 86 +++++++++++++++++++++--------------- v2/internal/runner/runner.go | 2 +- v2/pkg/core/execute.go | 1 + 4 files changed, 96 insertions(+), 66 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e5e74498f..bbce9b151 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,93 +5,99 @@ go 1.17 require ( github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible - github.com/akrylysov/pogreb v0.10.1 // indirect - github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c + github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.14.0 - github.com/antchfx/htmlquery v1.2.3 + github.com/antchfx/htmlquery v1.2.4 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 - github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 - github.com/go-rod/rod v0.101.7 + github.com/go-rod/rod v0.101.8 github.com/google/go-github v17.0.0+incompatible - github.com/gosuri/uilive v0.0.4 // indirect - github.com/gosuri/uiprogress v0.0.1 // indirect - github.com/itchyny/gojq v0.12.4 + github.com/itchyny/gojq v0.12.5 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/miekg/dns v1.1.43 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.11 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e + github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 - github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 + github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a github.com/projectdiscovery/gologger v1.1.4 - github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa + github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f github.com/projectdiscovery/interactsh v0.0.6 - github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 + github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 - github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a + github.com/projectdiscovery/retryabledns v1.0.12 github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d + github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.21.7 + github.com/shirou/gopsutil/v3 v3.21.9 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 - github.com/xanzy/go-gitlab v0.50.3 - github.com/ysmood/gson v0.6.4 // indirect - github.com/ysmood/leakless v0.7.0 // indirect + github.com/xanzy/go-gitlab v0.51.1 go.uber.org/atomic v1.9.0 go.uber.org/multierr v1.7.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/net v0.0.0-20211020060615-d418f374d309 + golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/text v0.3.7 - google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.8 // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect + github.com/akrylysov/pogreb v0.10.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect - github.com/antchfx/xpath v1.1.6 // indirect - github.com/aymerick/douceur v0.2.0 // indirect + github.com/antchfx/xpath v1.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect + github.com/c4milo/unpackit v0.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/cockroachdb/errors v1.8.6 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb // indirect + github.com/cockroachdb/redact v1.1.3 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect + github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/css v1.0.0 // indirect + github.com/gosuri/uilive v0.0.4 // indirect + github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -100,8 +106,10 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-isatty v0.0.13 // indirect - github.com/microcosm-cc/bluemonday v1.0.15 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -111,17 +119,22 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect - github.com/tklauser/go-sysconf v0.3.7 // indirect - github.com/tklauser/numcpus v0.2.3 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/tklauser/numcpus v0.3.0 // indirect github.com/trivago/tgo v1.0.7 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.3.0 // indirect + github.com/ysmood/gson v0.6.4 // indirect + github.com/ysmood/leakless v0.7.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/exp v0.0.0-20210826195003-46c773283d9d // indirect + golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/v2/go.sum b/v2/go.sum index b95089c7e..6b184d9f4 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -35,12 +35,14 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7 git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= @@ -67,8 +69,9 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= +github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -79,10 +82,12 @@ github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5z github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= -github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -96,14 +101,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -124,8 +126,10 @@ github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -140,13 +144,18 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.8.6 h1:Am9evxl/po3RzpokemQvq7S7Cd0mxv24xy0B/trlQF4= github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb h1:ZkYzWJexCQZyR1ejLy1IHvgdXTMe3Hiyvwlkyk5xr5Y= github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -172,12 +181,15 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= @@ -185,6 +197,7 @@ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -216,6 +229,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -232,8 +246,8 @@ github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= -github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= -github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= +github.com/go-rod/rod v0.101.8/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -247,11 +261,13 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -328,8 +344,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -385,8 +399,9 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= -github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= +github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0= +github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= @@ -479,8 +494,6 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= -github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -555,6 +568,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -572,9 +586,8 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542 h1:wvHtITg+Y2eMVdN/4OR8wx7747RFTZX/UcxxJOE35F4= github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -584,16 +597,16 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= +github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f h1:lLBOzyKoIZTUoMIi3hxa8fj6EtK0EozlftfR4A8Mc4w= github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= -github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c= github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= @@ -609,23 +622,21 @@ github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 h1:SNtAiRRrJtDJJDroaa/bFXt/Tix2LA6+rHRib0ORlJQ= -github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df h1:CvTNAUD5JbLMqpMFoGNgfk2gOcN0NC57ICu0+oK84vs= +github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df/go.mod h1:pxWVDgq88t9dWv4+J2AIaWgY+EqOE1AyfHS0Tn23w4M= github.com/projectdiscovery/nuclei/v2 v2.5.1/go.mod h1:sU2qcY0MQFS0CqP1BgkR8ZnUyFhqK0BdnY6bvTKNjXY= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= +github.com/projectdiscovery/retryabledns v1.0.12 h1:OzCsUaipN75OwjtH62FxBIhKye1NmnfG4DxtQclOtns= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= -github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -664,8 +675,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -674,8 +683,9 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg= +github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -726,10 +736,12 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= -github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py53eg= github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= -github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= +github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= +github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= +github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= @@ -752,8 +764,9 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= -github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xanzy/go-gitlab v0.51.1 h1:wWKLalwx4omxFoHh3PLs9zDgAD4GXDP/uoxwMRCSiWM= +github.com/xanzy/go-gitlab v0.51.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -846,6 +859,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20210826195003-46c773283d9d h1:0jjV4zcNl3aKvtqwdWq0orBLGpQB9TZ9qH45oQhI5MM= golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -922,16 +936,17 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= +golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab h1:llrcWN/wOwO+6gAyfBzxb5hZ+c3mriU/0+KNgYu6adA= golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 h1:B333XXssMuKQeBwiNODx4TupZy7bf4sxFZnN2ZOcvUE= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1006,9 +1021,10 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 84f2837a8..4f8ef9753 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -18,7 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" - "github.com/projectdiscovery/nuclei/v2/pkg/engine/inputs/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 3593123a2..883707875 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -49,6 +49,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu // All other request types are executed here e.executeModelWithInput(templateType, template, input, results) } + wg.Done() } e.workPool.Wait() return results From 533fb3f1080a395a7799969dfc5ff71ca0cdaa93 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 28 Oct 2021 22:02:22 +0530 Subject: [PATCH 051/196] misc flag update --- v2/cmd/nuclei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef65582..85f0ecf35 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,8 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "pt", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "ept", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) From 1ca2cf3beace4bb5b876b50e8b13a4289c7091f9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 28 Oct 2021 23:17:05 +0530 Subject: [PATCH 052/196] Misc --- v2/internal/runner/runner.go | 118 +++++++++++++++----------------- v2/pkg/catalog/loader/loader.go | 21 ++++++ 2 files changed, 75 insertions(+), 64 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4f8ef9753..d27713a35 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -267,22 +267,7 @@ func (r *Runner) RunEnumeration() error { } executerOpts.WorkflowLoader = workflowLoader - loaderConfig := loader.Config{ - Templates: r.options.Templates, - Workflows: r.options.Workflows, - ExcludeTemplates: r.options.ExcludedTemplates, - Tags: r.options.Tags, - ExcludeTags: r.options.ExcludeTags, - IncludeTemplates: r.options.IncludeTemplates, - Authors: r.options.Author, - Severities: r.options.Severities, - ExcludeSeverities: r.options.ExcludeSeverities, - IncludeTags: r.options.IncludeTags, - TemplatesDirectory: r.options.TemplatesDirectory, - Catalog: r.catalog, - ExecutorOptions: executerOpts, - } - store, err := loader.New(&loaderConfig) + store, err := loader.New(loader.NewConfig(r.options, r.catalog, executerOpts)) if err != nil { return errors.Wrap(err, "could not load templates from config") } @@ -300,54 +285,7 @@ func (r *Runner) RunEnumeration() error { return nil // exit } - // Display stats for any loaded templates' syntax warnings or errors - stats.Display(parsers.SyntaxWarningStats) - stats.Display(parsers.SyntaxErrorStats) - - builder := &strings.Builder{} - if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { - builder.WriteString(" (") - - if strings.Contains(config.Version, "-dev") { - builder.WriteString(r.colorizer.Blue("development").String()) - } else if config.Version == r.templatesConfig.NucleiLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr := builder.String() - builder.Reset() - - gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) - - if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic - builder.WriteString(" (") - - if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { - builder.WriteString(r.colorizer.Green("latest").String()) - } else { - builder.WriteString(r.colorizer.Red("outdated").String()) - } - builder.WriteString(")") - } - messageStr = builder.String() - builder.Reset() - - if r.templatesConfig != nil { - gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) - } - if r.interactsh != nil { - gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) - } - if len(store.Templates()) > 0 { - gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) - gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) - } - if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) - } + r.displayExecutionInfo(store) var unclusteredRequests int64 for _, template := range store.Templates() { @@ -417,6 +355,58 @@ func (r *Runner) RunEnumeration() error { return nil } +// displayExecutionInfo displays misc info about the nuclei engine execution +func (r *Runner) displayExecutionInfo(store *loader.Store) { + // Display stats for any loaded templates' syntax warnings or errors + stats.Display(parsers.SyntaxWarningStats) + stats.Display(parsers.SyntaxErrorStats) + + builder := &strings.Builder{} + if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { + builder.WriteString(" (") + + if strings.Contains(config.Version, "-dev") { + builder.WriteString(r.colorizer.Blue("development").String()) + } else if config.Version == r.templatesConfig.NucleiLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr := builder.String() + builder.Reset() + + gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) + + if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { // TODO extract duplicated logic + builder.WriteString(" (") + + if r.templatesConfig.TemplateVersion == r.templatesConfig.NucleiTemplatesLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr = builder.String() + builder.Reset() + + if r.templatesConfig != nil { + gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.TemplateVersion, messageStr) + } + if r.interactsh != nil { + gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) + } + if len(store.Templates()) > 0 { + gologger.Info().Msgf("Templates added in last update: %d", r.countNewTemplates()) + gologger.Info().Msgf("Templates loaded for scan: %d", len(store.Templates())) + } + if len(store.Workflows()) > 0 { + gologger.Info().Msgf("Workflows loaded for scan: %d", len(store.Workflows())) + } +} + // readNewTemplatesFile reads newly added templates from directory if it exists func (r *Runner) readNewTemplatesFile() ([]string, error) { if r.templatesConfig == nil { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4ae..386898b12 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Config contains the configuration options for the loader @@ -44,6 +45,26 @@ type Store struct { preprocessor templates.Preprocessor } +// NewConfig returns a new loader config +func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts protocols.ExecuterOptions) *Config { + loaderConfig := Config{ + Templates: options.Templates, + Workflows: options.Workflows, + ExcludeTemplates: options.ExcludedTemplates, + Tags: options.Tags, + ExcludeTags: options.ExcludeTags, + IncludeTemplates: options.IncludeTemplates, + Authors: options.Author, + Severities: options.Severities, + ExcludeSeverities: options.ExcludeSeverities, + IncludeTags: options.IncludeTags, + TemplatesDirectory: options.TemplatesDirectory, + Catalog: catalog, + ExecutorOptions: executerOpts, + } + return &loaderConfig +} + // New creates a new template store based on provided configuration func New(config *Config) (*Store, error) { // Create a tag filter based on provided configuration From 5393cc4cd509c95eda6dc7e543f5384feec09268 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 29 Oct 2021 03:19:43 +0530 Subject: [PATCH 053/196] Adjusting packages for more API-type design --- v2/internal/testutils/testutils.go | 38 +------------------ v2/pkg/core/inputs/inputs.go | 17 +++++++++ v2/pkg/output/output.go | 35 +++++++++++++++++ v2/pkg/progress/progress.go | 24 ++++++++++++ .../protocols/common/interactsh/interactsh.go | 14 +++++++ v2/pkg/types/types.go | 14 +++++++ 6 files changed, 105 insertions(+), 37 deletions(-) create mode 100644 v2/pkg/core/inputs/inputs.go diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630..e58c70791 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -1,7 +1,6 @@ package testutils import ( - "github.com/logrusorgru/aurora" "go.uber.org/ratelimit" "github.com/projectdiscovery/gologger/levels" @@ -60,41 +59,6 @@ var DefaultOptions = &types.Options{ CustomHeaders: []string{}, } -// MockOutputWriter is a mocked output writer. -type MockOutputWriter struct { - aurora aurora.Aurora - RequestCallback func(templateID, url, requestType string, err error) - WriteCallback func(o *output.ResultEvent) -} - -// NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} -} - -// Close closes the output writer interface -func (m *MockOutputWriter) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *MockOutputWriter) Colorizer() aurora.Aurora { - return m.aurora -} - -// Write writes the event to file and/or screen. -func (m *MockOutputWriter) Write(result *output.ResultEvent) error { - if m.WriteCallback != nil { - m.WriteCallback(result) - } - return nil -} - -// Request writes a log the requests trace log -func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { - if m.RequestCallback != nil { - m.RequestCallback(templateID, url, requestType, err) - } -} - // TemplateInfo contains info for a mock executed template. type TemplateInfo struct { ID string @@ -109,7 +73,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: NewMockOutputWriter(), + Output: output.NewMockOutputWriter(), Options: options, Progress: progressImpl, ProjectFile: nil, diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go new file mode 100644 index 000000000..6237dfb99 --- /dev/null +++ b/v2/pkg/core/inputs/inputs.go @@ -0,0 +1,17 @@ +package inputs + +type SimpleInputProvider struct { + Inputs []string +} + +// Count returns the number of items for input provider +func (s *SimpleInputProvider) Count() int64 { + return int64(len(s.Inputs)) +} + +// Scan calls a callback function till the input provider is exhausted +func (s *SimpleInputProvider) Scan(callback func(value string)) { + for _, v := range s.Inputs { + callback(v) + } +} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index b90be29c3..26555719d 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -209,3 +209,38 @@ func (w *StandardWriter) Close() { w.traceFile.Close() } } + +// MockOutputWriter is a mocked output writer. +type MockOutputWriter struct { + aurora aurora.Aurora + RequestCallback func(templateID, url, requestType string, err error) + WriteCallback func(o *ResultEvent) +} + +// NewMockOutputWriter creates a new mock output writer +func NewMockOutputWriter() *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false)} +} + +// Close closes the output writer interface +func (m *MockOutputWriter) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *MockOutputWriter) Colorizer() aurora.Aurora { + return m.aurora +} + +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) Write(result *ResultEvent) error { + if m.WriteCallback != nil { + m.WriteCallback(result) + } + return nil +} + +// Request writes a log the requests trace log +func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { + if m.RequestCallback != nil { + m.RequestCallback(templateID, url, requestType, err) + } +} diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 220ff49a6..fcd34340d 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -247,3 +247,27 @@ func (p *StatsTicker) Stop() { _ = p.server.Shutdown(context.Background()) } } + +type MockProgressClient struct{} + +// Stop stops the progress recorder. +func (m *MockProgressClient) Stop() {} + +// Init inits the progress bar with initial details for scan +func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} + +// AddToTotal adds a value to the total request count +func (m *MockProgressClient) AddToTotal(delta int64) {} + +// IncrementRequests increments the requests counter by 1. +func (m *MockProgressClient) IncrementRequests() {} + +// IncrementMatched increments the matched counter by 1. +func (m *MockProgressClient) IncrementMatched() {} + +// IncrementErrorsBy increments the error counter by count. +func (m *MockProgressClient) IncrementErrorsBy(count int64) {} + +// IncrementFailedRequestsBy increments the number of requests counter by count +// along with errors. +func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5c2914ff5..0ea42d460 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -103,6 +103,20 @@ func New(options *Options) (*Client, error) { return interactClient, nil } +// NewDefaultOptions returns the default options for interactsh client +func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { + return &Options{ + ServerURL: "https://interactsh.com", + CacheSize: 5000, + Eviction: 60 * time.Second, + ColldownPeriod: 5 * time.Second, + PollDuration: 5 * time.Second, + Output: output, + IssuesClient: reporting, + Progress: progress, + } +} + func (c *Client) firstTimeInitializeClient() error { interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9f8f784b1..7f5a1851b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -181,3 +181,17 @@ func (options *Options) AddVarPayload(key string, value interface{}) { func (options *Options) VarsPayload() map[string]interface{} { return options.varsPayload } + +// DefaultOptions returns default options for nuclei +func DefaultOptions() *Options { + return &Options{ + RateLimit: 150, + BulkSize: 25, + TemplateThreads: 25, + HeadlessBulkSize: 10, + HeadlessTemplateThreads: 10, + Timeout: 5, + Retries: 1, + MaxHostError: 30, + } +} From 75f18f169ca35be5ea603599a662d4ec7b2d4510 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 29 Oct 2021 18:30:24 +0530 Subject: [PATCH 054/196] Adding response highlighting to websocket --- v2/pkg/protocols/others/ssl/ssl.go | 2 +- v2/pkg/protocols/others/websocket/websocket.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index ec72fcc91..ec6a32a18 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -96,7 +96,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {}) + event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) callback(event) return nil } diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index a2487aa0a..7c8558540 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -22,6 +22,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -278,12 +279,6 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Output.Request(r.options.TemplateID, input, "websocket", err) gologger.Verbose().Msgf("Sent Websocket request to %s", input) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responseOutput) - } - data := make(map[string]interface{}) for k, v := range previous { data[k] = v @@ -303,6 +298,11 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } callback(event) return nil From 3a47413cd484eb5173d821981009fbd4c7000c2d Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:08:18 +0300 Subject: [PATCH 055/196] Add case-insensitive flag to matchers and extractors --- v2/pkg/operators/extractors/compile.go | 10 ++++++++++ v2/pkg/operators/extractors/extract.go | 12 +++++++++++- v2/pkg/operators/extractors/extractors.go | 7 +++++++ v2/pkg/operators/matchers/compile.go | 10 ++++++++++ v2/pkg/operators/matchers/match.go | 4 ++++ v2/pkg/operators/matchers/matchers.go | 6 ++++++ 6 files changed, 48 insertions(+), 1 deletion(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index e688c9520..07306e207 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -46,5 +46,15 @@ func (e *Extractor) CompileExtractors() error { if e.Part == "" { e.Part = "body" } + + if e.CaseInsensitive { + if e.Type != "kval" { + return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) + } + for i := range e.KVal { + e.KVal[i] = strings.ToLower(e.KVal[i]) + } + } + return nil } diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 72440c6ec..e43ea145e 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -34,8 +34,18 @@ func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { // ExtractKval extracts key value pairs from a data map func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} { - results := make(map[string]struct{}) + if e.CaseInsensitive { + inputData := data + data = make(map[string]interface{}, len(inputData)) + for k, v := range inputData { + if s, ok := v.(string); ok { + v = strings.ToLower(s) + } + data[strings.ToLower(k)] = v + } + } + results := make(map[string]struct{}) for _, k := range e.KVal { item, ok := data[k] if !ok { diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 5c126a271..d4b2bb3c4 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -105,6 +105,13 @@ type Extractor struct { // Internal, when set to true will allow using the value extracted // in the next request for some protocols (like HTTP). Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"` + + // description: | + // CaseInsensitive enables case-insensitive extractions. Default is false. + // values: + // - false + // - true + CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` } // ExtractorType is the type of the extractor specified diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 09213e23a..63b3edf58 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "fmt" "regexp" + "strings" "github.com/Knetic/govaluate" @@ -60,5 +61,14 @@ func (m *Matcher) CompileMatchers() error { } else { m.condition = ORCondition } + + if m.CaseInsensitive { + if m.Type != "word" { + return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) + } + for i := range m.Words { + m.Words[i] = strings.ToLower(m.Words[i]) + } + } return nil } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index fff0055de..df71a1442 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -42,6 +42,10 @@ func (m *Matcher) MatchSize(length int) bool { // MatchWords matches a word check against a corpus. func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + if m.CaseInsensitive { + corpus = strings.ToLower(corpus) + } + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 88ee413e1..594a6fc1d 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -105,6 +105,12 @@ type Matcher struct { // values: // - "hex" Encoding string `yaml:"encoding,omitempty" jsonschema:"title=encoding for word field,description=Optional encoding for the word fields,enum=hex"` + // description: | + // CaseInsensitive enables case-insensitive matches. Default is false. + // values: + // - false + // - true + CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive match,description=use case insensitive match"` // cached data for the compiled matcher condition ConditionType From bfb69b2ff2fca56e354598c03c91f5044ed658d9 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:10:17 +0300 Subject: [PATCH 056/196] Extract getMatchPart method in protocols --- v2/pkg/protocols/dns/operators.go | 37 +++++++++++++------------- v2/pkg/protocols/file/operators.go | 33 ++++++++++++----------- v2/pkg/protocols/headless/operators.go | 33 ++++++++++++----------- v2/pkg/protocols/http/operators.go | 6 ++--- v2/pkg/protocols/network/operators.go | 33 ++++++++++++----------- 5 files changed, 73 insertions(+), 69 deletions(-) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index b9f0454b3..0cf54988e 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -16,13 +16,7 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "": - partString = "raw" - } - - item, ok := data[partString] + item, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } @@ -50,25 +44,32 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - part := extractor.Part + item, ok := request.getMatchPart(extractor.Part, data) + if !ok { + return nil + } + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(types.ToString(item)) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +func (request *Request) getMatchPart(part string, data output.InternalEvent) (interface{}, bool) { switch part { - case "body", "all": + case "body", "all", "": part = "raw" } item, ok := data[part] if !ok { - return nil + return "", false } - itemStr := types.ToString(item) - switch extractor.GetType() { - case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) - case extractors.KValExtractor: - return extractor.ExtractKval(data) - } - return nil + return item, true } // responseToDSLMap converts a DNS response to a map for use in DSL matching diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index c0d95c5b0..cde2e63e0 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -16,17 +16,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "data", "": - partString = "raw" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -45,17 +38,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "all", "data", "": - partString = "raw" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -66,6 +52,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "all", "data", "": + part = "raw" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a file response to a map for use in DSL matching func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent { return output.InternalEvent{ diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 81a81a6ab..3d62e61b5 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -14,17 +14,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "resp", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -43,17 +36,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "resp", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -64,6 +50,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "resp", "": + part = "data" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a headless response to a map for use in DSL matching func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { return output.InternalEvent{ diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 9ae939cf6..8d7c4b72a 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -17,7 +17,7 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - item, ok := getMatchPart(matcher.Part, data) + item, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } @@ -57,7 +57,7 @@ func getStatusCode(data map[string]interface{}) (int, bool) { // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := getMatchPart(extractor.Part, data) + item, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } @@ -75,7 +75,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } // getMatchPart returns the match part honoring "all" matchers + others. -func getMatchPart(part string, data output.InternalEvent) (string, bool) { +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { if part == "header" { part = "all_headers" } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 8e0330dc0..c829b8ade 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -14,17 +14,10 @@ import ( // Match matches a generic data response again a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - partString := matcher.Part - switch partString { - case "body", "all", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(matcher.Part, data) if !ok { return false, []string{} } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: @@ -43,17 +36,10 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat // Extract performs extracting operation for an extractor on model and returns true or false. func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "body", "all", "": - partString = "data" - } - - item, ok := data[partString] + itemStr, ok := request.getMatchPart(extractor.Part, data) if !ok { return nil } - itemStr := types.ToString(item) switch extractor.GetType() { case extractors.RegexExtractor: @@ -64,6 +50,21 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto return nil } +func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + switch part { + case "body", "all", "": + part = "data" + } + + item, ok := data[part] + if !ok { + return "", false + } + itemStr := types.ToString(item) + + return itemStr, true +} + // responseToDSLMap converts a network response to a map for use in DSL matching func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { return output.InternalEvent{ From 897f11213c16d6a45b4ff78befa74edc17cbd296 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:11:09 +0300 Subject: [PATCH 057/196] Add unit tests for case-insensitive flag in protocols --- v2/pkg/protocols/dns/operators_test.go | 24 +++++++++++++++++ v2/pkg/protocols/file/operators_test.go | 20 ++++++++++++++ v2/pkg/protocols/http/operators_test.go | 31 ++++++++++++++++++++++ v2/pkg/protocols/network/operators_test.go | 19 +++++++++++++ 4 files changed, 94 insertions(+) diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index c8d06124a..6bb91b7e2 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -134,6 +134,30 @@ func TestDNSOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + req := new(dns.Msg) + req.Question = append(req.Question, dns.Question{Name: "ONE.ONE.ONE.ONE.", Qtype: dns.TypeA, Qclass: dns.ClassINET}) + + resp := new(dns.Msg) + resp.Rcode = dns.RcodeSuccess + resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "ONE.ONE.ONE.ONE."}}) + + event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE") + + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"one.ONE.one.ONE"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, []string{"one.one.one.one"}, matched) + }) } func TestDNSOperatorExtract(t *testing.T) { diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index b03d641b5..da40c76d4 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -105,6 +105,26 @@ func TestFileOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + resp := "TEST-DATA\r\n1.1.1.1\r\n" + event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") + require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Equal(t, resp, event["raw"], "could not get correct resp") + + matcher := &matchers.Matcher{ + Part: "raw", + Type: "word", + Words: []string{"TeSt-DaTA"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"test-data"}, matched) + }) } func TestFileOperatorExtract(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index c9d3b795d..e2ecbdf57 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -117,6 +117,21 @@ func TestHTTPOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "body", + Type: "word", // only applies to word + Words: []string{"EXAMPLE DOMAIN"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"example domain"}, matched) + }) } func TestHTTPOperatorExtract(t *testing.T) { @@ -215,6 +230,22 @@ func TestHTTPOperatorExtract(t *testing.T) { require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data") }) }) + + t.Run("caseInsensitive", func(t *testing.T) { + event["body"] = exampleResponseBody + + extractor := &extractors.Extractor{ + Type: "kval", + KVal: []string{"TEST_HEADER"}, // only applies to KVal + CaseInsensitive: true, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile kval extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor kval valid response") + require.Equal(t, map[string]struct{}{"test-response": {}}, data, "could not extract correct kval data") + }) } func TestHTTPMakeResult(t *testing.T) { diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index bf577d7e7..735d9af38 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -103,6 +103,25 @@ func TestNetworkOperatorMatch(t *testing.T) { require.False(t, isMatched, "could match invalid response matcher") require.Equal(t, []string{}, matched) }) + + t.Run("caseInsensitive", func(t *testing.T) { + matcher := &matchers.Matcher{ + Part: "body", + Type: "word", + Words: []string{"rESp-DAta"}, + CaseInsensitive: true, + } + err = matcher.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + req := "TEST-DATA\r\n" + resp := "RESP-DATA\r\nSTAT \r\n" + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "TEST") + + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, []string{"resp-data"}, matched) + }) } func TestNetworkOperatorExtract(t *testing.T) { From 392ea23f1d0b97d95492ed173b9958fa11a56d62 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Fri, 29 Oct 2021 19:11:57 +0300 Subject: [PATCH 058/196] Add integration tests for case-insensitive flag --- .../http/get-case-insensitive.yaml | 16 ++++ v2/cmd/integration-test/http.go | 82 +++++++++++++++---- 2 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 integration_tests/http/get-case-insensitive.yaml diff --git a/integration_tests/http/get-case-insensitive.yaml b/integration_tests/http/get-case-insensitive.yaml new file mode 100644 index 000000000..e8c4054b7 --- /dev/null +++ b/integration_tests/http/get-case-insensitive.yaml @@ -0,0 +1,16 @@ +id: basic-get-case-insensitive + +info: + name: Basic GET Request + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + case-insensitive: true + words: + - "ThIS is TEsT MAtcHEr TExT" diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c8bc6acc1..a81111c4c 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -15,24 +15,26 @@ import ( ) var httpTestcases = map[string]testutils.TestCase{ - "http/get-headers.yaml": &httpGetHeaders{}, - "http/get-query-string.yaml": &httpGetQueryString{}, - "http/get-redirects.yaml": &httpGetRedirects{}, - "http/get.yaml": &httpGet{}, - "http/post-body.yaml": &httpPostBody{}, - "http/post-json-body.yaml": &httpPostJSONBody{}, - "http/post-multipart-body.yaml": &httpPostMultipartBody{}, - "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, - "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, - "http/raw-get-query.yaml": &httpRawGetQuery{}, - "http/raw-get.yaml": &httpRawGet{}, - "http/raw-payload.yaml": &httpRawPayload{}, - "http/raw-post-body.yaml": &httpRawPostBody{}, - "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, - "http/request-condition.yaml": &httpRequestCondition{}, - "http/request-condition-new.yaml": &httpRequestCondition{}, - "http/interactsh.yaml": &httpInteractshRequest{}, - "http/self-contained.yaml": &httpRequestSelContained{}, + "http/get-headers.yaml": &httpGetHeaders{}, + "http/get-query-string.yaml": &httpGetQueryString{}, + "http/get-redirects.yaml": &httpGetRedirects{}, + "http/get.yaml": &httpGet{}, + "http/post-body.yaml": &httpPostBody{}, + "http/post-json-body.yaml": &httpPostJSONBody{}, + "http/post-multipart-body.yaml": &httpPostMultipartBody{}, + "http/raw-cookie-reuse.yaml": &httpRawCookieReuse{}, + "http/raw-dynamic-extractor.yaml": &httpRawDynamicExtractor{}, + "http/raw-get-query.yaml": &httpRawGetQuery{}, + "http/raw-get.yaml": &httpRawGet{}, + "http/raw-payload.yaml": &httpRawPayload{}, + "http/raw-post-body.yaml": &httpRawPostBody{}, + "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, + "http/request-condition.yaml": &httpRequestCondition{}, + "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, + "http/self-contained.yaml": &httpRequestSelContained{}, + "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, + "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, } type httpInteractshRequest struct{} @@ -553,3 +555,47 @@ func (h *httpRequestSelContained) Execute(filePath string) error { } return nil } + +type httpGetCaseInsensitive struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetCaseInsensitive) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "THIS IS TEST MATCHER TEXT") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type httpGetCaseInsensitiveCluster struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + files := strings.Split(filesPath, ",") + + results, err := testutils.RunNucleiTemplateAndGetResults(files[0], ts.URL, debug, "-t", files[1]) + if err != nil { + return err + } + if len(results) != 2 { + return errIncorrectResultsCount(results) + } + return nil +} From 04e3c0165a31965bbafd46f3a4e2ec8c61680fd3 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Sat, 30 Oct 2021 13:17:47 +0300 Subject: [PATCH 059/196] feat: In case of binary data, show a hexadecimal view as well #1080 --- v2/cmd/nuclei/main.go | 7 -- .../helpers/responsehighlighter/hexdump.go | 116 ++++++++++++++++++ .../response_highlighter.go | 17 ++- .../response_highlighter_test.go | 78 ++++++++++++ v2/pkg/protocols/dns/request.go | 27 ++-- v2/pkg/protocols/file/request.go | 23 +++- v2/pkg/protocols/headless/request.go | 34 ++--- v2/pkg/protocols/http/request.go | 35 +++++- v2/pkg/protocols/network/request.go | 16 ++- 9 files changed, 303 insertions(+), 50 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go create mode 100644 v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..32e6221f8 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -175,10 +175,3 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags currentFlag.Group(groupName) } } - -/* -HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass. - -https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest - -*/ diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go new file mode 100644 index 000000000..10c2cfc5c --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go @@ -0,0 +1,116 @@ +package responsehighlighter + +import ( + "errors" + "fmt" + "regexp" + "strings" + "unicode" + + "github.com/projectdiscovery/gologger" +) + +// [0-9a-fA-F]{8} {2} - hexdump indexes (8 character hex value followed by two spaces) +// [0-9a-fA-F]{2} + - 2 character long hex values followed by one or two space (potentially wrapped with an ASCII color code, see below) +// \x1b\[(\d;?)+m - ASCII color code pattern +// \x1b\[0m - ASCII color code reset +// \|(.*)\|\n - ASCII representation of the input delimited by pipe characters +var hexDumpParsePattern = regexp.MustCompile(`([0-9a-fA-F]{8} {2})((?:(?:\x1b\[(?:\d;?)+m)?[0-9a-fA-F]{2}(?:\x1b\[0m)? +)+)\|(.*)\|\n`) +var hexValuePattern = regexp.MustCompile(`([a-fA-F0-9]{2})`) + +type HighlightableHexDump struct { + index []string + hex []string + ascii []string +} + +func NewHighlightableHexDump(rowSize int) HighlightableHexDump { + return HighlightableHexDump{index: make([]string, 0, rowSize), hex: make([]string, 0, rowSize), ascii: make([]string, 0, rowSize)} +} + +func (hexDump HighlightableHexDump) len() int { + return len(hexDump.index) +} + +func (hexDump HighlightableHexDump) String() string { + var result string + for i := 0; i < hexDump.len(); i++ { + result += hexDump.index[i] + hexDump.hex[i] + "|" + hexDump.ascii[i] + "|\n" + } + return result +} + +func toHighLightedHexDump(hexDump, snippetToHighlight string) (HighlightableHexDump, error) { + hexDumpRowValues := hexDumpParsePattern.FindAllStringSubmatch(hexDump, -1) + if hexDumpRowValues == nil || len(hexDumpRowValues) != strings.Count(hexDump, "\n") { + message := "could not parse hexdump" + gologger.Warning().Msgf(message) + return HighlightableHexDump{}, errors.New(message) + } + + result := NewHighlightableHexDump(len(hexDumpRowValues)) + for _, currentHexDumpRowValues := range hexDumpRowValues { + result.index = append(result.index, currentHexDumpRowValues[1]) + result.hex = append(result.hex, currentHexDumpRowValues[2]) + result.ascii = append(result.ascii, currentHexDumpRowValues[3]) + } + return result.highlight(snippetToHighlight), nil +} + +func (hexDump HighlightableHexDump) highlight(snippetToColor string) HighlightableHexDump { + return highlightAsciiSection(highlightHexSection(hexDump, snippetToColor), snippetToColor) +} + +func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { + var snippetHexCharactersMatchPattern string + for _, char := range snippetToColor { + snippetHexCharactersMatchPattern += fmt.Sprintf(`(%02x[ \n]+)`, char) + } + + hexDump.hex = highlight(hexDump.hex, snippetHexCharactersMatchPattern, func(v string) string { + return hexValuePattern.ReplaceAllString(v, addColor("$1")) + }) + + return hexDump +} + +func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { + var snippetCharactersMatchPattern string + for _, v := range snippetToColor { + snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v))) + } + + hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string { + if len(v) > 1 { + return addColor(string(v[0])) + v[1:] // do not color new line characters + } + return addColor(v) + }) + + return hexDump +} + +func highlight(values []string, snippetCharactersMatchPattern string, replaceToFunc func(v string) string) []string { + rows := strings.Join(values, "\n") + compiledPattern := regexp.MustCompile(snippetCharactersMatchPattern) + for _, submatch := range compiledPattern.FindAllStringSubmatch(rows, -1) { + var replaceTo string + var replaceFrom string + for _, matchedValueWithSuffix := range submatch[1:] { + replaceFrom += matchedValueWithSuffix + replaceTo += replaceToFunc(matchedValueWithSuffix) + } + rows = strings.ReplaceAll(rows, replaceFrom, replaceTo) + } + return strings.Split(rows, "\n") +} + +// IsASCII tests whether a string consists only of ASCII characters or not +func IsASCII(input string) bool { + for i := 0; i < len(input); i++ { + if input[i] > unicode.MaxASCII { + return false + } + } + return true +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go index 1cb914e97..f637644f0 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -9,15 +9,22 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" ) -var colorizer = aurora.NewAurora(true) +var colorFunction = aurora.Green -func Highlight(operatorResult *operators.Result, response string, noColor bool) string { +func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string { result := response if operatorResult != nil && !noColor { for _, matches := range operatorResult.Matches { if len(matches) > 0 { for _, currentMatch := range matches { - result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + if hexDump { + highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) + if err == nil { + result = highlightedHexDump.String() + } + } else { + result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch)) + } } } } @@ -33,3 +40,7 @@ func CreateStatusCodeSnippet(response string, statusCode int) string { } return "" } + +func addColor(value string) string { + return colorFunction(value).String() +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go new file mode 100644 index 000000000..a60341ef1 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go @@ -0,0 +1,78 @@ +package responsehighlighter + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" +) + +const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn" + +func TestHexDumpHighlighting(t *testing.T) { + const highlightedHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| +00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| +00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| +00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| +00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| +00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| +00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| +00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| +` + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { + result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij") + assert.Nil(t, err) + assert.Equal(t, highlightedHexDumpResponse, result.String()) + }) + + t.Run("Test highlight when the snippet contains separator character", func(t *testing.T) { + value := "asdfasdfasda|basdfadsdfs|" + result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b") + + expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 61 7c 62 61 73 |asdfasdfasda|bas| +00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs|| +` + assert.Nil(t, err) + assert.Equal(t, expected, result.String()) + }) +} + +func TestHighlight(t *testing.T) { + const multiSnippetHighlightHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| +00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| +00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| +00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| +00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| +00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| +00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| +00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| +` + matches := map[string][]string{ + "first": {"defghij"}, + "second": {"ab"}, + } + operatorResult := operators.Result{Matches: matches} + + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { + result := Highlight(&operatorResult, hex.Dump([]byte(input)), false, true) + assert.Equal(t, multiSnippetHighlightHexDumpResponse, result) + }) + + t.Run("Test highlighting without hexdump", func(t *testing.T) { + result := Highlight(&operatorResult, input, false, false) + expected := `abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn` + assert.Equal(t, expected, result) + }) + + t.Run("Test the response is not modified if noColor is true", func(t *testing.T) { + result := Highlight(&operatorResult, input, true, false) + assert.Equal(t, input, result) + }) + + t.Run("Test the response is not modified if noColor is true", func(t *testing.T) { + result := Highlight(&operatorResult, hex.Dump([]byte(input)), true, true) + assert.Equal(t, hex.Dump([]byte(input)), result) + }) +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 28a3309b0..61927ea44 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -1,6 +1,7 @@ package dns import ( + "encoding/hex" "net/url" "github.com/pkg/errors" @@ -44,35 +45,45 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // Send the request to the target servers - resp, err := request.dnsClient.Do(compiledRequest) + response, err := request.dnsClient.Do(compiledRequest) if err != nil { request.options.Output.Request(request.options.TemplateID, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) } - if resp == nil { + if response == nil { return errors.Wrap(err, "could not send dns request") } request.options.Progress.IncrementRequests() request.options.Output.Request(request.options.TemplateID, domain, "dns", err) - gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) - outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor)) - } + debug(event, request, domain, response.String()) callback(event) return nil } +func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain) + + hexDump := false + if !responsehighlighter.IsASCII(response) { + hexDump = true + response = hex.Dump([]byte(response)) + } + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + } +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 61bb87185..e27331edc 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -1,6 +1,8 @@ package file import ( + "encoding/hex" + "fmt" "io/ioutil" "os" @@ -49,20 +51,17 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } - dataStr := tostring.UnsafeToString(buffer) + fileContent := tostring.UnsafeToString(buffer) gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath) - outputEvent := request.responseToDSLMap(dataStr, input, filePath) + outputEvent := request.responseToDSLMap(fileContent, input, filePath) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor)) - } + debug(event, request, filePath, fileContent) callback(event) }(data) @@ -76,3 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review request.options.Progress.IncrementRequests() return nil } + +func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + hexDump := false + if !responsehighlighter.IsASCII(fileContent) { + hexDump = true + fileContent = hex.Dump([]byte(fileContent)) + } + logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath) + gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump)) + } +} diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 55394ed7f..0486009ac 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -17,42 +17,42 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer instance.Close() - parsed, err := url.Parse(input) + parsedURL, err := url.Parse(inputURL) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } - out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) + out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Output.Request(request.options.TemplateID, inputURL, "headless", nil) request.options.Progress.IncrementRequests() - gologger.Verbose().Msgf("Sent Headless request to %s", input) + gologger.Verbose().Msgf("Sent Headless request to %s", inputURL) reqBuilder := &strings.Builder{} if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input) + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, inputURL) for _, act := range request.Steps { reqBuilder.WriteString(act.String()) reqBuilder.WriteString("\n") } - gologger.Print().Msgf("%s", reqBuilder.String()) + gologger.Print().Msgf(reqBuilder.String()) } var responseBody string @@ -60,18 +60,22 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp if err == nil { responseBody, _ = html.HTML() } - outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL) for k, v := range out { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor)) - } + debug(event, request, responseBody, inputURL) callback(event) return nil } + +func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false)) + } +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3451e5f3d..da8558831 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -2,6 +2,7 @@ package http import ( "bytes" + "encoding/hex" "fmt" "io" "io/ioutil" @@ -450,7 +451,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // Decode gbk response content-types // gb18030 supersedes gb2312 - if isContentTypeGbk(resp.Header.Get("Content-Type")) { + responseContentType := resp.Header.Get("Content-Type") + if isContentTypeGbk(responseContentType) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") @@ -509,10 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor)) - } + debug(request, formedURL, redirectedResponse, responseContentType, event) callback(event) return nil @@ -532,3 +531,29 @@ func (request *Request) setCustomHeaders(req *generatedRequest) { } } } + +const CRLF = "\r\n" + +func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + hexDump := false + response := string(redirectedResponse) + + var headers string + if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) { + hexDump = true + responseLines := strings.Split(response, CRLF) + for i, value := range responseLines { + headers += value + CRLF + if value == "" { + response = hex.Dump([]byte(strings.Join(responseLines[i+1:], ""))) + break + } + } + } + logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL) + + gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false)) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + } +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 66da31e9b..12e34e8dd 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -190,8 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.options.Options.Debug || request.options.Options.DebugRequests { requestOutput := reqBuilder.String() - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput))) } request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) @@ -274,14 +274,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input }) } - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor)) - } + debug(event, request, response, actualAddress) return nil } +func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true)) + } +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { From 3f1186da2bfb740eec30e74ebcd2134aed0e1de6 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:39:38 +0300 Subject: [PATCH 060/196] Add error log support --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 2 +- v2/pkg/output/output.go | 40 ++++++++++++++++++++++++++++-------- v2/pkg/types/types.go | 2 ++ 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..da5a1ddbf 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -139,6 +139,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), + flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f806952..ca74f6a49 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -215,7 +215,7 @@ func New(options *types.Options) (*Runner, error) { } // Create the output file if asked - outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile) + outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile, options.ErrorLogFile) if err != nil { return nil, errors.Wrap(err, "could not create output file") } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index b90be29c3..572045f5a 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -41,6 +41,8 @@ type StandardWriter struct { outputMutex *sync.Mutex traceFile *fileWriter traceMutex *sync.Mutex + errorFile *fileWriter + errorMutex *sync.Mutex severityColors func(severity.Severity) string } @@ -97,7 +99,7 @@ type ResultEvent struct { } // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) var outputFile *fileWriter @@ -116,6 +118,14 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } traceOutput = output } + var errorOutput *fileWriter + if errorFile != "" { + output, err := newFileOutputWriter(errorFile) + if err != nil { + return nil, errors.Wrap(err, "could not create error file") + } + errorOutput = output + } writer := &StandardWriter{ json: json, jsonReqResp: jsonReqResp, @@ -126,6 +136,8 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, outputMutex: &sync.Mutex{}, traceFile: traceOutput, traceMutex: &sync.Mutex{}, + errorFile: errorOutput, + errorMutex: &sync.Mutex{}, severityColors: colorizer.New(auroraColorizer), } return writer, nil @@ -171,8 +183,8 @@ type JSONTraceRequest struct { } // Request writes a log the requests trace log -func (w *StandardWriter) Request(templateID, url, requestType string, err error) { - if w.traceFile == nil { +func (w *StandardWriter) Request(templateID, url, requestType string, requestErr error) { + if w.traceFile == nil && w.errorFile == nil { return } request := &JSONTraceRequest{ @@ -180,8 +192,8 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) URL: url, Type: requestType, } - if err != nil { - request.Error = err.Error() + if requestErr != nil { + request.Error = requestErr.Error() } else { request.Error = "none" } @@ -190,9 +202,18 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) if err != nil { return } - w.traceMutex.Lock() - _ = w.traceFile.Write(data) - w.traceMutex.Unlock() + + if w.traceFile != nil { + w.traceMutex.Lock() + _ = w.traceFile.Write(data) + w.traceMutex.Unlock() + } + + if requestErr != nil && w.errorFile != nil { + w.errorMutex.Lock() + _ = w.errorFile.Write(data) + w.errorMutex.Unlock() + } } // Colorizer returns the colorizer instance for writer @@ -208,4 +229,7 @@ func (w *StandardWriter) Close() { if w.traceFile != nil { w.traceFile.Close() } + if w.errorFile != nil { + w.errorFile.Close() + } } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..dcab9eae2 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -57,6 +57,8 @@ type Options struct { TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests TraceLogFile string + // ErrorLogFile specifies a file to write with the errors of all requests + ErrorLogFile string // ReportingDB is the db for report storage as well as deduplication ReportingDB string // ReportingConfig is the config file for nuclei reporting module From 933ed2429d533f7d78ce30df4eee51ade0313be9 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:46:26 +0300 Subject: [PATCH 061/196] Update json log request format --- v2/pkg/output/output.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 572045f5a..57c841a8a 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -174,23 +174,23 @@ func (w *StandardWriter) Write(event *ResultEvent) error { return nil } -// JSONTraceRequest is a trace log request written to file -type JSONTraceRequest struct { - ID string `json:"id"` - URL string `json:"url"` - Error string `json:"error"` - Type string `json:"type"` +// JSONLogRequest is a trace/error log request written to file +type JSONLogRequest struct { + Template string `json:"template"` + Input string `json:"input"` + Error string `json:"error"` + Type string `json:"type"` } // Request writes a log the requests trace log -func (w *StandardWriter) Request(templateID, url, requestType string, requestErr error) { +func (w *StandardWriter) Request(templatePath, input, requestType string, requestErr error) { if w.traceFile == nil && w.errorFile == nil { return } - request := &JSONTraceRequest{ - ID: templateID, - URL: url, - Type: requestType, + request := &JSONLogRequest{ + Template: templatePath, + Input: input, + Type: requestType, } if requestErr != nil { request.Error = requestErr.Error() From 463c1c0142968de19e97c9d18667acc1c74bbf71 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 12:55:02 +0300 Subject: [PATCH 062/196] Use template path in output request --- v2/pkg/protocols/dns/request.go | 6 +++--- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/headless/request.go | 8 ++++---- v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/network/request.go | 18 +++++++++--------- v2/pkg/protocols/offlinehttp/request.go | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 28a3309b0..8980aeb65 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -28,7 +28,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Compile each request for the template based on the URL compiledRequest, err := request.Make(domain) if err != nil { - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } @@ -46,7 +46,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Send the request to the target servers resp, err := request.dnsClient.Do(compiledRequest) if err != nil { - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) request.options.Progress.IncrementFailedRequestsBy(1) } if resp == nil { @@ -54,7 +54,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } request.options.Progress.IncrementRequests() - request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 61bb87185..c1dfe4a25 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -69,7 +69,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 55394ed7f..35bdf28bd 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -20,7 +20,7 @@ var _ protocols.Request = &Request{} func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } @@ -28,19 +28,19 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp parsed, err := url.Parse(input) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Output.Request(request.options.TemplatePath, input, "headless", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Output.Request(request.options.TemplatePath, input, "headless", nil) request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", input) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3451e5f3d..3dc5d3be9 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -356,7 +356,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send @@ -394,7 +394,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) - request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) duration := time.Since(timeStart) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 66da31e9b..6c2fe6619 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -36,7 +36,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review address, err = getAddress(input) } if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "network", err) + request.options.Output.Request(request.options.TemplatePath, input, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -65,7 +65,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return err } @@ -113,7 +113,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } @@ -143,7 +143,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input data = []byte(input.Data) } if err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -151,7 +151,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", dataErr) + request.options.Output.Request(request.options.TemplatePath, address, "network", dataErr) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -162,7 +162,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input return nil } if _, err := conn.Write(finalData); err != nil { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -194,7 +194,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) } - request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplatePath, actualAddress, "network", err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 @@ -225,7 +225,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } @@ -238,7 +238,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input final = make([]byte, bufferSize) n, err = conn.Read(final) if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 1eb4bfc57..c9dbc1a82 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -91,7 +91,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } From 1eb0378952b576db290fa15fc87fc6904ace13e5 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 13:46:07 +0300 Subject: [PATCH 063/196] Unwrap errors in json log output --- v2/pkg/output/output.go | 5 +++-- v2/pkg/utils/utils.go | 12 ++++++++++++ v2/pkg/utils/utils_test.go | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/utils/utils_test.go diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 57c841a8a..0b0cbf028 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -192,8 +193,8 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques Input: input, Type: requestType, } - if requestErr != nil { - request.Error = requestErr.Error() + if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil { + request.Error = unwrappedErr.Error() } else { request.Error = "none" } diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go index 8d6fc0cf7..ee9247618 100644 --- a/v2/pkg/utils/utils.go +++ b/v2/pkg/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "errors" "strings" ) @@ -11,3 +12,14 @@ func IsBlank(value string) bool { func IsNotBlank(value string) bool { return !IsBlank(value) } + +func UnwrapError(err error) error { + for { // get the last wrapped error + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + break + } + err = unwrapped + } + return err +} diff --git a/v2/pkg/utils/utils_test.go b/v2/pkg/utils/utils_test.go new file mode 100644 index 000000000..ede9d530e --- /dev/null +++ b/v2/pkg/utils/utils_test.go @@ -0,0 +1,21 @@ +package utils + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnwrapError(t *testing.T) { + require.Equal(t, nil, UnwrapError(nil)) + + errOne := fmt.Errorf("error one") + require.Equal(t, errOne, UnwrapError(errOne)) + + errTwo := fmt.Errorf("error with error: %w", errOne) + require.Equal(t, errOne, UnwrapError(errTwo)) + + errThree := fmt.Errorf("error with error: %w", errTwo) + require.Equal(t, errOne, UnwrapError(errThree)) +} From cd2db280bfd5469f5984d1d3f1bb96828cbdb6c8 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 14:13:35 +0300 Subject: [PATCH 064/196] Move output mutex to fileWriter --- v2/pkg/output/file_output_writer.go | 6 ++++++ v2/pkg/output/output.go | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index 94765ae23..84df0ab5f 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -2,11 +2,13 @@ package output import ( "os" + "sync" ) // fileWriter is a concurrent file based output writer. type fileWriter struct { file *os.File + mu sync.Mutex } // NewFileOutputWriter creates a new buffered writer for a file @@ -20,6 +22,8 @@ func newFileOutputWriter(file string) (*fileWriter, error) { // WriteString writes an output to the underlying file func (w *fileWriter) Write(data []byte) error { + w.mu.Lock() + defer w.mu.Unlock() if _, err := w.file.Write(data); err != nil { return err } @@ -29,6 +33,8 @@ func (w *fileWriter) Write(data []byte) error { // Close closes the underlying writer flushing everything to disk func (w *fileWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() //nolint:errcheck // we don't care whether sync failed or succeeded. w.file.Sync() return w.file.Close() diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 0b0cbf028..7189beb48 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -3,7 +3,6 @@ package output import ( "os" "regexp" - "sync" "time" "github.com/pkg/errors" @@ -39,11 +38,8 @@ type StandardWriter struct { noMetadata bool aurora aurora.Aurora outputFile *fileWriter - outputMutex *sync.Mutex traceFile *fileWriter - traceMutex *sync.Mutex errorFile *fileWriter - errorMutex *sync.Mutex severityColors func(severity.Severity) string } @@ -134,11 +130,8 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, noTimestamp: noTimestamp, aurora: auroraColorizer, outputFile: outputFile, - outputMutex: &sync.Mutex{}, traceFile: traceOutput, - traceMutex: &sync.Mutex{}, errorFile: errorOutput, - errorMutex: &sync.Mutex{}, severityColors: colorizer.New(auroraColorizer), } return writer, nil @@ -205,15 +198,11 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques } if w.traceFile != nil { - w.traceMutex.Lock() _ = w.traceFile.Write(data) - w.traceMutex.Unlock() } if requestErr != nil && w.errorFile != nil { - w.errorMutex.Lock() _ = w.errorFile.Write(data) - w.errorMutex.Unlock() } } From b8ebbc27f5a11f6d5edd74a5415bc0041893850e Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 14:19:11 +0300 Subject: [PATCH 065/196] Use io.WriteCloser in output writer --- v2/pkg/output/file_output_writer.go | 10 ++++++---- v2/pkg/output/output.go | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index 84df0ab5f..8c1ab0ebe 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -21,14 +21,16 @@ func newFileOutputWriter(file string) (*fileWriter, error) { } // WriteString writes an output to the underlying file -func (w *fileWriter) Write(data []byte) error { +func (w *fileWriter) Write(data []byte) (int, error) { w.mu.Lock() defer w.mu.Unlock() if _, err := w.file.Write(data); err != nil { - return err + return 0, err } - _, err := w.file.Write([]byte("\n")) - return err + if _, err := w.file.Write([]byte("\n")); err != nil { + return 0, err + } + return len(data) + 1, nil } // Close closes the underlying writer flushing everything to disk diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 7189beb48..991c99232 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -1,6 +1,7 @@ package output import ( + "io" "os" "regexp" "time" @@ -37,9 +38,9 @@ type StandardWriter struct { noTimestamp bool noMetadata bool aurora aurora.Aurora - outputFile *fileWriter - traceFile *fileWriter - errorFile *fileWriter + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser severityColors func(severity.Severity) string } @@ -99,7 +100,7 @@ type ResultEvent struct { func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) - var outputFile *fileWriter + var outputFile io.WriteCloser if file != "" { output, err := newFileOutputWriter(file) if err != nil { @@ -107,7 +108,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } outputFile = output } - var traceOutput *fileWriter + var traceOutput io.WriteCloser if traceFile != "" { output, err := newFileOutputWriter(traceFile) if err != nil { @@ -115,7 +116,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, } traceOutput = output } - var errorOutput *fileWriter + var errorOutput io.WriteCloser if errorFile != "" { output, err := newFileOutputWriter(errorFile) if err != nil { @@ -161,7 +162,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if !w.json { data = decolorizerRegex.ReplaceAll(data, []byte("")) } - if writeErr := w.outputFile.Write(data); writeErr != nil { + if _, writeErr := w.outputFile.Write(data); writeErr != nil { return errors.Wrap(err, "could not write to output") } } @@ -198,11 +199,11 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques } if w.traceFile != nil { - _ = w.traceFile.Write(data) + _, _ = w.traceFile.Write(data) } if requestErr != nil && w.errorFile != nil { - _ = w.errorFile.Write(data) + _, _ = w.errorFile.Write(data) } } From bccc8e921b9aa640fd244f9d149ace0723e704ab Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 30 Oct 2021 15:04:16 +0300 Subject: [PATCH 066/196] Add test for output --- v2/pkg/output/output_test.go | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 v2/pkg/output/output_test.go diff --git a/v2/pkg/output/output_test.go b/v2/pkg/output/output_test.go new file mode 100644 index 000000000..9f1d188bb --- /dev/null +++ b/v2/pkg/output/output_test.go @@ -0,0 +1,59 @@ +package output + +import ( + "fmt" + "strings" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestStandardWriterRequest(t *testing.T) { + t.Run("WithoutTraceAndError", func(t *testing.T) { + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + require.NoError(t, err) + require.NotPanics(t, func() { + w.Request("path", "input", "http", nil) + w.Close() + }) + }) + + t.Run("TraceAndErrorWithoutError", func(t *testing.T) { + traceWriter := &testWriteCloser{} + errorWriter := &testWriteCloser{} + + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w.traceFile = traceWriter + w.errorFile = errorWriter + require.NoError(t, err) + w.Request("path", "input", "http", nil) + + require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String()) + require.Empty(t, errorWriter.String()) + }) + + t.Run("ErrorWithWrappedError", func(t *testing.T) { + errorWriter := &testWriteCloser{} + + w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w.errorFile = errorWriter + require.NoError(t, err) + w.Request( + "misconfiguration/tcpconfig.yaml", + "https://example.com/tcpconfig.html", + "http", + fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")), + ) + + require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String()) + }) +} + +type testWriteCloser struct { + strings.Builder +} + +func (w testWriteCloser) Close() error { + return nil +} From 38f2cf245e36e1139aa0b0683479c767828494f9 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Sat, 30 Oct 2021 16:41:10 +0300 Subject: [PATCH 067/196] feat: In case of binary data, show a hexadecimal view as well #1080 * added Compact hex view to the output if the -vv flag is provided --- v2/pkg/protocols/network/request.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 12e34e8dd..c7c31e07b 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -189,9 +189,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - requestOutput := reqBuilder.String() gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s", hex.Dump([]byte(requestOutput))) + requestBytes := []byte(reqBuilder.String()) + gologger.Print().Msgf("%s", hex.Dump(requestBytes)) + if request.options.Options.VerboseVerbose { + gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes)) + } } request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) @@ -282,7 +285,18 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { if request.options.Options.Debug || request.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), request.options.Options.NoColor, true)) + requestBytes := []byte(response) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true)) + if request.options.Options.VerboseVerbose { + var allMatches []string + for _, namedMatch := range event.OperatorsResult.Matches { + for _, matchElement := range namedMatch { + allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) + } + } + event.OperatorsResult.Matches["compact"] = allMatches + gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false)) + } } } From e1c39f255cdc674fde32a8a3fa750e7e21d25e45 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 1 Nov 2021 09:46:34 +0000 Subject: [PATCH 068/196] Auto Generate Syntax Docs + JSONSchema [Mon Nov 1 09:46:34 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 40 +++++++++++++++++++++++++++++++ nuclei-jsonschema.json | 10 ++++++++ v2/pkg/templates/templates_doc.go | 22 +++++++++++++++-- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index a3fffe351..3d16bda71 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1554,6 +1554,26 @@ Valid values:
+
+ +case-insensitive bool + +
+
+ +CaseInsensitive enables case-insensitive matches. Default is false. + + +Valid values: + + + - false + + - true +
+ +
+ @@ -1833,6 +1853,26 @@ in the next request for some protocols (like HTTP).
+
+ +case-insensitive bool + +
+
+ +CaseInsensitive enables case-insensitive extractions. Default is false. + + +Valid values: + + + - false + + - true +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1db207d77..4f22c3863 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -191,6 +191,11 @@ "type": "boolean", "title": "mark extracted value for internal variable use", "description": "Internal when set to true will allow using the value extracted in the next request for some protocols" + }, + "case-insensitive": { + "type": "boolean", + "title": "use case insensitive extract", + "description": "use case insensitive extract" } }, "additionalProperties": false, @@ -293,6 +298,11 @@ "type": "string", "title": "encoding for word field", "description": "Optional encoding for the word fields" + }, + "case-insensitive": { + "type": "boolean", + "title": "use case insensitive match", + "description": "use case insensitive match" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d6042dfbd..ca416b3b8 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -473,7 +473,7 @@ func init() { FieldName: "matchers", }, } - MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 12) + MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" MATCHERSMatcherDoc.Fields[0].Type = "string" MATCHERSMatcherDoc.Fields[0].Note = "" @@ -575,6 +575,15 @@ func init() { MATCHERSMatcherDoc.Fields[11].Values = []string{ "hex", } + MATCHERSMatcherDoc.Fields[12].Name = "case-insensitive" + MATCHERSMatcherDoc.Fields[12].Type = "bool" + MATCHERSMatcherDoc.Fields[12].Note = "" + MATCHERSMatcherDoc.Fields[12].Description = "CaseInsensitive enables case-insensitive matches. Default is false." + MATCHERSMatcherDoc.Fields[12].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive matches. Default is false." + MATCHERSMatcherDoc.Fields[12].Values = []string{ + "false", + "true", + } EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." @@ -601,7 +610,7 @@ func init() { FieldName: "extractors", }, } - EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 10) + EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11) EXTRACTORSExtractorDoc.Fields[0].Name = "name" EXTRACTORSExtractorDoc.Fields[0].Type = "string" EXTRACTORSExtractorDoc.Fields[0].Note = "" @@ -678,6 +687,15 @@ func init() { EXTRACTORSExtractorDoc.Fields[9].Note = "" EXTRACTORSExtractorDoc.Fields[9].Description = "Internal, when set to true will allow using the value extracted\nin the next request for some protocols (like HTTP)." EXTRACTORSExtractorDoc.Fields[9].Comments[encoder.LineComment] = "Internal, when set to true will allow using the value extracted" + EXTRACTORSExtractorDoc.Fields[10].Name = "case-insensitive" + EXTRACTORSExtractorDoc.Fields[10].Type = "bool" + EXTRACTORSExtractorDoc.Fields[10].Note = "" + EXTRACTORSExtractorDoc.Fields[10].Description = "CaseInsensitive enables case-insensitive extractions. Default is false." + EXTRACTORSExtractorDoc.Fields[10].Comments[encoder.LineComment] = "CaseInsensitive enables case-insensitive extractions. Default is false." + EXTRACTORSExtractorDoc.Fields[10].Values = []string{ + "false", + "true", + } DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" From bb05be7b952e595662db7415d6a9124fdcc8a7ea Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 15:47:20 +0530 Subject: [PATCH 069/196] Added integration tests for websocket + misc fixes --- integration_tests/websocket/basic.yaml | 16 ++++ integration_tests/websocket/cswsh.yaml | 16 ++++ integration_tests/websocket/no-cswsh.yaml | 16 ++++ v2/cmd/integration-test/integration-test.go | 9 +- v2/cmd/integration-test/websocket.go | 90 +++++++++++++++++++ v2/go.mod | 1 + v2/go.sum | 2 + v2/internal/testutils/integration.go | 24 +++++ .../protocols/others/websocket/websocket.go | 16 ++-- v2/pkg/templates/templates.go | 6 ++ 10 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 integration_tests/websocket/basic.yaml create mode 100644 integration_tests/websocket/cswsh.yaml create mode 100644 integration_tests/websocket/no-cswsh.yaml create mode 100644 v2/cmd/integration-test/websocket.go diff --git a/integration_tests/websocket/basic.yaml b/integration_tests/websocket/basic.yaml new file mode 100644 index 000000000..c09378301 --- /dev/null +++ b/integration_tests/websocket/basic.yaml @@ -0,0 +1,16 @@ +id: basic-request + +info: + name: Basic Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + inputs: + - data: hello + matchers: + - type: word + words: + - world + part: response \ No newline at end of file diff --git a/integration_tests/websocket/cswsh.yaml b/integration_tests/websocket/cswsh.yaml new file mode 100644 index 000000000..80f75d7ac --- /dev/null +++ b/integration_tests/websocket/cswsh.yaml @@ -0,0 +1,16 @@ +id: basic-cswsh-request + +info: + name: Basic cswsh Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + headers: + Origin: 'http://evil.com' + matchers: + - type: word + words: + - true + part: success \ No newline at end of file diff --git a/integration_tests/websocket/no-cswsh.yaml b/integration_tests/websocket/no-cswsh.yaml new file mode 100644 index 000000000..6833d804e --- /dev/null +++ b/integration_tests/websocket/no-cswsh.yaml @@ -0,0 +1,16 @@ +id: basic-nocswsh-request + +info: + name: Basic Non-Vulnerable cswsh Request + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + headers: + Origin: 'http://evil.com' + matchers: + - type: word + words: + - true + part: success \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index baa743e8d..41c035f60 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -22,10 +22,11 @@ func main() { failed := aurora.Red("[✘]").String() protocolTests := map[string]map[string]testutils.TestCase{ - "http": httpTestcases, - "network": networkTestcases, - "dns": dnsTestCases, - "workflow": workflowTestcases, + "http": httpTestcases, + "network": networkTestcases, + "dns": dnsTestCases, + "workflow": workflowTestcases, + "websocket": websocketTestCases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go new file mode 100644 index 000000000..25f249af9 --- /dev/null +++ b/v2/cmd/integration-test/websocket.go @@ -0,0 +1,90 @@ +package main + +import ( + "net" + "strings" + + "github.com/gobwas/ws/wsutil" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" +) + +var websocketTestCases = map[string]testutils.TestCase{ + "websocket/basic.yaml": &websocketBasic{}, + "websocket/cswsh.yaml": &websocketCswsh{}, + "websocket/no-cswsh.yaml": &websocketNoCswsh{}, +} + +type websocketBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketBasic) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + for { + msg, op, _ := wsutil.ReadClientData(conn) + if string(msg) != string("hello") { + return + } + _ = wsutil.WriteServerMessage(conn, op, []byte("world")) + } + } + originValidate := func(origin string) bool { + return true + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketCswsh struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketCswsh) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return true + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketNoCswsh struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketNoCswsh) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return origin == "https://google.com" + } + ts := testutils.NewWebsocketServer(connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 0 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/go.mod b/v2/go.mod index 9d3d5017a..3596218c3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -100,6 +100,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect diff --git a/v2/go.sum b/v2/go.sum index d4a21e8b5..74aba21ba 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -556,6 +556,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 8c5491d28..13a3d8c69 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -4,10 +4,14 @@ import ( "errors" "fmt" "net" + "net/http" + "net/http/httptest" "os" "os/exec" "regexp" "strings" + + "github.com/gobwas/ws" ) // RunNucleiTemplateAndGetResults returns a list of results for a template @@ -113,3 +117,23 @@ func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer { func (s *TCPServer) Close() { s.listener.Close() } + +// NewWebsocketServer creates a new websocket server from a handler +func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { + w.WriteHeader(http.StatusBadRequest) + return + } + conn, _, _, err := ws.UpgradeHTTP(r, w) + if err != nil { + return + } + go func() { + defer conn.Close() + + handler(conn) + }() + })) + return ts +} diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go index 7c8558540..ec73e8274 100644 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ b/v2/pkg/protocols/others/websocket/websocket.go @@ -99,7 +99,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Payloads) > 0 { attackType := r.AttackType if attackType == "" { - attackType = "sniper" + attackType = "batteringram" } r.attackType = generators.StringToType[attackType] @@ -183,10 +183,10 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu if err != nil { return errors.Wrap(err, "could not parse input url") } - payloadValues["Address"] = parsed.Host + payloadValues["Hostname"] = parsed.Host + payloadValues["Host"] = parsed.Hostname() payloadValues["Scheme"] = parsed.Scheme payloadValues["Path"] = parsed.Path - payloadValues["hostname"] = parsed.Hostname() for key, value := range r.Headers { finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) @@ -210,10 +210,14 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + addressToDial := string(finalAddress) + if parsed.Path != "" && parsed.Path != "/" { + addressToDial = addressToDial + parsed.Path + } - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), string(finalAddress)) + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "ssl", err) + r.options.Output.Request(r.options.TemplateID, input, "websocket", err) r.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } @@ -296,7 +300,7 @@ func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValu data["ip"] = r.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues + internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues }) if r.options.Options.Debug || r.options.Options.DebugResponse { responseOutput := responseBuilder.String() diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index f71fa1dfa..79e6d766d 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -90,6 +90,8 @@ var TemplateTypes = []string{ "headless", "network", "workflow", + "ssl", + "websocket", } // Type returns the type of the template @@ -107,6 +109,10 @@ func (t *Template) Type() string { return "network" case len(t.Workflows) > 0: return "workflow" + case len(t.RequestsSSL) > 0: + return "ssl" + case len(t.RequestsWebsocket) > 0: + return "websocket" default: return "" } From a274cc572254d8d0404a21726271b90b0efe7b62 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 15:51:56 +0530 Subject: [PATCH 070/196] Misc integration test --- integration_tests/websocket/path.yaml | 16 ++++++++++++++ v2/cmd/integration-test/websocket.go | 30 ++++++++++++++++++++++++--- v2/internal/testutils/integration.go | 19 ++++++++++++++--- 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 integration_tests/websocket/path.yaml diff --git a/integration_tests/websocket/path.yaml b/integration_tests/websocket/path.yaml new file mode 100644 index 000000000..d3607b30a --- /dev/null +++ b/integration_tests/websocket/path.yaml @@ -0,0 +1,16 @@ +id: basic-request-path + +info: + name: Basic Request Path + author: pdteam + severity: info + +websocket: + - address: '{{Scheme}}://{{Hostname}}' + inputs: + - data: hello + matchers: + - type: word + words: + - world + part: response \ No newline at end of file diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index 25f249af9..be20ba773 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -12,6 +12,7 @@ var websocketTestCases = map[string]testutils.TestCase{ "websocket/basic.yaml": &websocketBasic{}, "websocket/cswsh.yaml": &websocketCswsh{}, "websocket/no-cswsh.yaml": &websocketNoCswsh{}, + "websocket/path.yaml": &websocketWithPath{}, } type websocketBasic struct{} @@ -30,7 +31,7 @@ func (h *websocketBasic) Execute(filePath string) error { originValidate := func(origin string) bool { return true } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) @@ -53,7 +54,7 @@ func (h *websocketCswsh) Execute(filePath string) error { originValidate := func(origin string) bool { return true } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) @@ -76,7 +77,30 @@ func (h *websocketNoCswsh) Execute(filePath string) error { originValidate := func(origin string) bool { return origin == "https://google.com" } - ts := testutils.NewWebsocketServer(connHandler, originValidate) + ts := testutils.NewWebsocketServer("", connHandler, originValidate) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) + if err != nil { + return err + } + if len(results) != 0 { + return errIncorrectResultsCount(results) + } + return nil +} + +type websocketWithPath struct{} + +// Execute executes a test case and returns an error if occurred +func (h *websocketWithPath) Execute(filePath string) error { + connHandler := func(conn net.Conn) { + + } + originValidate := func(origin string) bool { + return origin == "https://google.com" + } + ts := testutils.NewWebsocketServer("/test", connHandler, originValidate) defer ts.Close() results, err := testutils.RunNucleiTemplateAndGetResults(filePath, strings.ReplaceAll(ts.URL, "http", "ws"), debug) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 13a3d8c69..4c3f84e33 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/gobwas/ws" + "github.com/julienschmidt/httprouter" ) // RunNucleiTemplateAndGetResults returns a list of results for a template @@ -119,8 +120,8 @@ func (s *TCPServer) Close() { } // NewWebsocketServer creates a new websocket server from a handler -func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate func(origin string) bool, port ...int) *httptest.Server { + handlerFunc := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if value := r.Header.Get("Origin"); value != "" && !originValidate(value) { w.WriteHeader(http.StatusBadRequest) return @@ -134,6 +135,18 @@ func NewWebsocketServer(handler func(conn net.Conn), originValidate func(origin handler(conn) }() - })) + }) + + var router *httprouter.Router + if path != "" { + router = httprouter.New() + router.HandlerFunc("*", "/test", handlerFunc) + } + var ts *httptest.Server + if router != nil { + ts = httptest.NewServer(router) + } else { + ts = httptest.NewServer(handlerFunc) + } return ts } From f8c6f0ba647aa37c3634e259751c09e405098dc7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 1 Nov 2021 18:02:45 +0530 Subject: [PATCH 071/196] Added debug for SSL --- v2/pkg/protocols/others/ssl/ssl.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/others/ssl/ssl.go index ec6a32a18..f8f34b00b 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/others/ssl/ssl.go @@ -8,7 +8,9 @@ import ( "strings" "time" + jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -17,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -71,6 +74,7 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu hostname, _, _ := net.SplitHostPort(address) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} + conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "ssl", err) @@ -87,16 +91,33 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu r.options.Output.Request(r.options.TemplateID, address, "ssl", err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if len(connTLS.ConnectionState().PeerCertificates) == 0 { + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", r.options.TemplateID, input) + } + + state := connTLS.ConnectionState() + if len(state.PeerCertificates) == 0 { return nil } + + tlsData := cryptoutil.TLSGrab(&state) + jsonData, _ := jsoniter.Marshal(tlsData) + jsonDataString := string(jsonData) + data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] + + data["response"] = jsonDataString data["host"] = input data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = r.dialer.GetDialedIP(hostname) event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := jsonDataString + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } callback(event) return nil } From b4775085215510402b2be6d9ec77b0f819146620 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:44:55 +0200 Subject: [PATCH 072/196] feat: In case of binary data, show a hexadecimal view as well #1080 * the ASCII column in the hex dump represents non-printable ASCII characters with a "." character, so in order to enable proper highlighting, those characters has to be replaced in the generated regex as well --- .../helpers/responsehighlighter/hexdump.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go index 10c2cfc5c..decc985c7 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/hexdump.go @@ -77,7 +77,13 @@ func highlightHexSection(hexDump HighlightableHexDump, snippetToColor string) Hi func highlightAsciiSection(hexDump HighlightableHexDump, snippetToColor string) HighlightableHexDump { var snippetCharactersMatchPattern string for _, v := range snippetToColor { - snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, regexp.QuoteMeta(string(v))) + var value string + if IsASCIIPrintable(v) { + value = regexp.QuoteMeta(string(v)) + } else { + value = "." + } + snippetCharactersMatchPattern += fmt.Sprintf(`(%s\n*)`, value) } hexDump.ascii = highlight(hexDump.ascii, snippetCharactersMatchPattern, func(v string) string { @@ -105,6 +111,10 @@ func highlight(values []string, snippetCharactersMatchPattern string, replaceToF return strings.Split(rows, "\n") } +func HasBinaryContent(input string) bool { + return !IsASCII(input) +} + // IsASCII tests whether a string consists only of ASCII characters or not func IsASCII(input string) bool { for i := 0; i < len(input); i++ { @@ -114,3 +124,7 @@ func IsASCII(input string) bool { } return true } + +func IsASCIIPrintable(input rune) bool { + return input > 32 && input < unicode.MaxASCII +} From 8f6280dc9dcd1bebebc09ec66cb3bb79c0ef0887 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:45:54 +0200 Subject: [PATCH 073/196] refactor: In case of binary data, show a hexadecimal view as well #1080 * small enhancements with regards to dumping responses --- v2/pkg/protocols/dns/request.go | 14 ++++----- v2/pkg/protocols/file/request.go | 14 ++++----- v2/pkg/protocols/headless/request.go | 11 +++---- v2/pkg/protocols/http/request.go | 43 ++++++++++++++++------------ v2/pkg/protocols/network/request.go | 41 +++++++++++++++----------- 5 files changed, 70 insertions(+), 53 deletions(-) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d063825f6..08422b8e7 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -65,22 +65,22 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, domain, response.String()) + dumpResponse(event, request.options, response.String(), domain) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, domain string, response string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n", request.options.TemplateID, domain) - +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(response) { + if responsehighlighter.HasBinaryContent(response) { hexDump = true response = hex.Dump([]byte(response)) } - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped DNS response for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) } } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index c7c3daee9..0326de377 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -2,7 +2,6 @@ package file import ( "encoding/hex" - "fmt" "io/ioutil" "os" @@ -61,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, filePath, fileContent) + dumpResponse(event, request.options, fileContent, filePath) callback(event) }(data) @@ -76,14 +75,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return nil } -func debug(event *output.InternalWrappedEvent, request *Request, filePath string, fileContent string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, fileContent string, filePath string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if !responsehighlighter.IsASCII(fileContent) { + if responsehighlighter.HasBinaryContent(fileContent) { hexDump = true fileContent = hex.Dump([]byte(fileContent)) } - logHeader := fmt.Sprintf("[%s] Dumped file request for %s\n", request.options.TemplateID, filePath) - gologger.Debug().Msgf("%s\n%s", logHeader, responsehighlighter.Highlight(event.OperatorsResult, fileContent, request.options.Options.NoColor, hexDump)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, fileContent, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped file request for %s\n\n%s", requestOptions.TemplateID, filePath, highlightedResponse) } } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 1609a595b..f4c0618b7 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -67,15 +67,16 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - debug(event, request, responseBody, inputURL) + dumpResponse(event, request.options, responseBody, inputURL) callback(event) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, responseBody string, input string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor, false)) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, responseBody string, input string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBody, cliOptions.NoColor, false) + gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse) } } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf5..28110d38e 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -511,7 +511,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) - debug(request, formedURL, redirectedResponse, responseContentType, event) + dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType) callback(event) return nil @@ -534,26 +534,33 @@ func (request *Request) setCustomHeaders(req *generatedRequest) { const CRLF = "\r\n" -func debug(request *Request, formedURL string, redirectedResponse []byte, responseContentType string, event *output.InternalWrappedEvent) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - hexDump := false +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, redirectedResponse []byte, formedURL string, responseContentType string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { response := string(redirectedResponse) - var headers string - if responseContentType == "" || responseContentType == "application/octet-stream" || (responseContentType == "application/x-www-form-urlencoded" && responsehighlighter.IsASCII(response)) { - hexDump = true - responseLines := strings.Split(response, CRLF) - for i, value := range responseLines { - headers += value + CRLF - if value == "" { - response = hex.Dump([]byte(strings.Join(responseLines[i+1:], ""))) - break - } - } + var highlightedResult string + if responseContentType == "application/octet-stream" || ((responseContentType == "" || responseContentType == "application/x-www-form-urlencoded") && responsehighlighter.HasBinaryContent(response)) { + highlightedResult = createResponseHexDump(event, response, cliOptions.NoColor) + } else { + highlightedResult = responsehighlighter.Highlight(event.OperatorsResult, response, cliOptions.NoColor, false) } - logMessageHeader := fmt.Sprintf("[%s] Dumped HTTP response for %s\n", request.options.TemplateID, formedURL) - gologger.Debug().Msgf("%s\n%s", logMessageHeader, responsehighlighter.Highlight(event.OperatorsResult, headers, request.options.Options.NoColor, false)) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, response, request.options.Options.NoColor, hexDump)) + gologger.Debug().Msgf("[%s] Dumped HTTP response for %s\n\n%s", requestOptions.TemplateID, formedURL, highlightedResult) + } +} + +func createResponseHexDump(event *output.InternalWrappedEvent, response string, noColor bool) string { + CRLFs := CRLF + CRLF + headerEndIndex := strings.Index(response, CRLFs) + len(CRLFs) + if headerEndIndex > 0 { + headers := response[0:headerEndIndex] + responseBodyHexDump := hex.Dump([]byte(response[headerEndIndex:])) + + highlightedHeaders := responsehighlighter.Highlight(event.OperatorsResult, headers, noColor, false) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, responseBodyHexDump, noColor, true) + return fmt.Sprintf("%s\n%s", highlightedHeaders, highlightedResponse) + } else { + return responsehighlighter.Highlight(event.OperatorsResult, hex.Dump([]byte(response)), noColor, true) } } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 761350856..fa1e1b60f 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -189,9 +190,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n", request.options.TemplateID, actualAddress) requestBytes := []byte(reqBuilder.String()) - gologger.Print().Msgf("%s", hex.Dump(requestBytes)) + gologger.Debug().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s\n%s", request.options.TemplateID, actualAddress, hex.Dump(requestBytes)) if request.options.Options.VerboseVerbose { gologger.Print().Msgf("\nCompact HEX view:\n%s", hex.EncodeToString(requestBytes)) } @@ -277,29 +277,38 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input }) } - debug(event, request, response, actualAddress) + dumpResponse(event, request.options, response, actualAddress) return nil } -func debug(event *output.InternalWrappedEvent, request *Request, response string, actualAddress string) { - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Network response for %s\n", request.options.TemplateID, actualAddress) +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, actualAddress string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { requestBytes := []byte(response) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), request.options.Options.NoColor, true)) - if request.options.Options.VerboseVerbose { - var allMatches []string - for _, namedMatch := range event.OperatorsResult.Matches { - for _, matchElement := range namedMatch { - allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) - } - } - event.OperatorsResult.Matches["compact"] = allMatches - gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor, false)) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, hex.Dump(requestBytes), cliOptions.NoColor, true) + gologger.Debug().Msgf("[%s] Dumped Network response for %s\n\n%s", requestOptions.TemplateID, actualAddress, highlightedResponse) + + if cliOptions.VerboseVerbose { + displayCompactHexView(event, response, cliOptions.NoColor) } } } +func displayCompactHexView(event *output.InternalWrappedEvent, response string, noColor bool) { + operatorsResult := event.OperatorsResult + if operatorsResult != nil { + var allMatches []string + for _, namedMatch := range operatorsResult.Matches { + for _, matchElement := range namedMatch { + allMatches = append(allMatches, hex.EncodeToString([]byte(matchElement))) + } + } + tempOperatorResult := &operators.Result{Matches: map[string][]string{"matchesInHex": allMatches}} + gologger.Print().Msgf("\nCompact HEX view:\n%s", responsehighlighter.Highlight(tempOperatorResult, hex.EncodeToString([]byte(response)), noColor, false)) + } +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { From 191797380e5385b39fe5a7e4a8288e34dca5848a Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 2 Nov 2021 14:12:59 +0530 Subject: [PATCH 074/196] bug: Bug in URL parsing for unsafe templates #830 Fixed raw request path not correct with unsafe --- v2/pkg/protocols/http/raw/raw.go | 23 +++++++++++++++++++---- v2/pkg/protocols/http/raw/raw_test.go | 11 +++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 2b3d44af5..18bebc39f 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -2,10 +2,12 @@ package raw import ( "bufio" + "bytes" "fmt" "io" "io/ioutil" "net/url" + "path/filepath" "strings" "github.com/projectdiscovery/rawhttp/client" @@ -27,6 +29,12 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { rawRequest := &Request{ Headers: make(map[string]string), } + + parsedURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + if unsafe { rawRequest.UnsafeRawBytes = []byte(request) } @@ -40,6 +48,11 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if len(parts) < 3 && !unsafe { return nil, fmt.Errorf("malformed request supplied") } + // Check if we have also a path from the passed base URL and if yes, + // append that to the unsafe request as well. + if parsedURL.Path != "" && strings.HasPrefix(parts[1], "/") && parts[1] != parsedURL.Path { + rawRequest.UnsafeRawBytes = fixUnsafeRequestPath(parsedURL, parts[1], rawRequest.UnsafeRawBytes) + } // Set the request Method rawRequest.Method = parts[0] @@ -98,10 +111,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { rawRequest.Path = parts[1] } - parsedURL, err := url.Parse(baseURL) - if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) - } hostURL := parsedURL.Host if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") { parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") @@ -131,3 +140,9 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } return rawRequest, nil } + +func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte { + fixedPath := filepath.Join(baseURL.Path, requestPath) + fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1) + return fixed +} diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index 540fb0178..0e7cee9b2 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -64,3 +64,14 @@ username=admin&password=login`, "https://test.com", false) require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly") require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly") } + +func TestParseUnsafeRequestWithPath(t *testing.T) { + request, err := Parse(`GET /manager/html HTTP/1.1 +Host: {{Hostname}} +Authorization: Basic {{base64('username:password')}} +User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0 +Accept-Language: en-US,en;q=0.9 +Connection: close`, "https://test.com/test/", true) + require.Nil(t, err, "could not parse unsafe request") + require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly") +} From 87c3071ba7cecb6708003310ccfc687444678bd7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 2 Nov 2021 14:59:05 +0530 Subject: [PATCH 075/196] Misc --- v2/go.mod | 4 ++++ v2/go.sum | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/v2/go.mod b/v2/go.mod index 3596218c3..df29aa9d3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -113,12 +113,16 @@ require ( github.com/trivago/tgo v1.0.7 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.3.0 // indirect github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect + github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect + github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 74aba21ba..1ff2440bd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -505,6 +505,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -680,6 +681,7 @@ github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bs github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -754,6 +756,8 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 h1:DyAZOw3JsVd6LJHqhl4MpKQdYQEmat0C6pPPwom39Ow= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= @@ -792,6 +796,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b h1:iYQzlljG1dOXBtsJGyzFC/wBK5qUCWs1eLCr/UcJYPA= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -839,6 +848,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -918,6 +929,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -1019,6 +1031,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 2224880d891f1e9a1dff4819c501f5d5c95436b9 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 2 Nov 2021 22:57:00 +0530 Subject: [PATCH 076/196] misc flag update --- v2/cmd/nuclei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 08905537b..7f43868f4 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -54,10 +54,10 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "templates", "Templates", flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.TemplateURLs, "template-urls", "tu", []string{}, "URL to a list of templates"), + flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "tu", []string{}, "URL containing list of templates to run"), flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-urls", "wu", []string{}, "URL to a list of workflows to run"), + flagSet.StringSliceVarP(&options.WorkflowURLs, "workflow-url", "wu", []string{}, "URL containing list of workflows to run"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) From bdb415b0c7f513bcb9acd7c5989b524b02aa38e3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 02:23:48 +0530 Subject: [PATCH 077/196] Misc changes as per review --- v2/cmd/nuclei/main.go | 2 +- v2/pkg/catalog/loader/loader.go | 2 +- v2/pkg/core/engine.go | 10 ++++++---- v2/pkg/parsers/workflow_loader.go | 2 +- v2/pkg/types/types.go | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 13602bfcd..08cc082a3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,7 +68,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.NormalizedStringSliceVarP(&options.Authors, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 386898b12..9915cf49e 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -54,7 +54,7 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr Tags: options.Tags, ExcludeTags: options.ExcludeTags, IncludeTemplates: options.IncludeTemplates, - Authors: options.Author, + Authors: options.Authors, Severities: options.Severities, ExcludeSeverities: options.ExcludeSeverities, IncludeTags: options.IncludeTags, diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index 455546244..5d7cf9461 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -5,7 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// Engine is an engine for running Nuclei Templates/Workflows. +// Engine is an executer for running Nuclei Templates/Workflows. // // The engine contains multiple thread pools which allow using different // concurrency values per protocol executed. @@ -19,14 +19,16 @@ type Engine struct { executerOpts protocols.ExecuterOptions } -// InputProvider is an input provider interface for the nuclei execution +// InputProvider is an input providing interface for the nuclei execution // engine. // -// An example InputProvider is provided in form of hmap input provider. +// An example InputProvider implementation is provided in form of hybrid +// input provider in pkg/core/inputs/hybrid/hmap.go type InputProvider interface { // Count returns the number of items for input provider Count() int64 - // Scan calls a callback function till the input provider is exhausted + // Scan iterates the input and for each found item calls the callback + // until the input provider has been exhausted. Scan(callback func(value string)) } diff --git a/v2/pkg/parsers/workflow_loader.go b/v2/pkg/parsers/workflow_loader.go index e3f9ab8b2..e0efd3fed 100644 --- a/v2/pkg/parsers/workflow_loader.go +++ b/v2/pkg/parsers/workflow_loader.go @@ -18,7 +18,7 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error) tagFilter := filter.New(&filter.Config{ Tags: options.Options.Tags, ExcludeTags: options.Options.ExcludeTags, - Authors: options.Options.Author, + Authors: options.Options.Authors, Severities: options.Options.Severities, IncludeTags: options.Options.IncludeTags, }) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 7f5a1851b..f618484eb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -29,8 +29,8 @@ type Options struct { Severities severity.Severities // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities - // Author filters templates based on their author and only run the matching ones. - Author goflags.NormalizedStringSlice + // Authors filters templates based on their author and only run the matching ones. + Authors goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist IncludeTags goflags.NormalizedStringSlice // IncludeTemplates includes specified templates to be run even while being in denylist From cf7628c45037dff4777f831d9ae1c02ed6d6724f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 02:34:48 +0530 Subject: [PATCH 078/196] Misc changes according to review --- v2/internal/testutils/integration.go | 12 +- v2/pkg/protocols/others/others.go | 1 - .../protocols/others/websocket/websocket.go | 365 ------------------ v2/pkg/protocols/{others => }/ssl/ssl.go | 62 +-- v2/pkg/protocols/{others => }/ssl/ssl_test.go | 0 v2/pkg/templates/templates.go | 4 +- 6 files changed, 36 insertions(+), 408 deletions(-) delete mode 100644 v2/pkg/protocols/others/others.go delete mode 100644 v2/pkg/protocols/others/websocket/websocket.go rename v2/pkg/protocols/{others => }/ssl/ssl.go (65%) rename v2/pkg/protocols/{others => }/ssl/ssl_test.go (100%) diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index 4c3f84e33..57fa1e0de 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -137,16 +137,10 @@ func NewWebsocketServer(path string, handler func(conn net.Conn), originValidate }() }) - var router *httprouter.Router if path != "" { - router = httprouter.New() + router := httprouter.New() router.HandlerFunc("*", "/test", handlerFunc) + return httptest.NewServer(router) } - var ts *httptest.Server - if router != nil { - ts = httptest.NewServer(router) - } else { - ts = httptest.NewServer(handlerFunc) - } - return ts + return httptest.NewServer(handlerFunc) } diff --git a/v2/pkg/protocols/others/others.go b/v2/pkg/protocols/others/others.go deleted file mode 100644 index 24b5bd240..000000000 --- a/v2/pkg/protocols/others/others.go +++ /dev/null @@ -1 +0,0 @@ -package others diff --git a/v2/pkg/protocols/others/websocket/websocket.go b/v2/pkg/protocols/others/websocket/websocket.go deleted file mode 100644 index ec73e8274..000000000 --- a/v2/pkg/protocols/others/websocket/websocket.go +++ /dev/null @@ -1,365 +0,0 @@ -package websocket - -import ( - "context" - "crypto/tls" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/gobwas/ws" - "github.com/gobwas/ws/wsutil" - "github.com/pkg/errors" - "github.com/projectdiscovery/fastdialer/fastdialer" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" -) - -// Request is a request for the Websocket protocol -type Request struct { - // Operators for the current request go here. - operators.Operators `yaml:",inline,omitempty"` - CompiledOperators *operators.Operators `yaml:"-"` - - // description: | - // Address contains address for the request - Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` - // description: | - // Inputs contains inputs for the websocket protocol - Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` - // description: | - // Headers contains headers for the request. - Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` - - // description: | - // Attack is the type of payload combinations to perform. - // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates - // permutations and combinations for all payloads. - // values: - // - "sniper" - // - "pitchfork" - // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` - // description: | - // Payloads contains any payloads for the current request. - // - // Payloads support both key-values combinations where a list - // of payloads is provided, or optionally a single file can also - // be provided as payload which will be read on run-time. - Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - - generator *generators.Generator - attackType generators.Type - - // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions -} - -// Input is an input for the websocket protocol -type Input struct { - // description: | - // Data is the data to send as the input. - // - // It supports DSL Helper Functions as well as normal expressions. - // examples: - // - value: "\"TEST\"" - // - value: "\"hex_decode('50494e47')\"" - Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` - // description: | - // Name is the optional name of the data read to provide matching on. - // examples: - // - value: "\"prefix\"" - Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` -} - -// Compile compiles the request generators preparing any requests possible. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - r.options = options - - client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) - if err != nil { - return errors.Wrap(err, "could not get network client") - } - r.dialer = client - - if len(r.Payloads) > 0 { - attackType := r.AttackType - if attackType == "" { - attackType = "batteringram" - } - r.attackType = generators.StringToType[attackType] - - // Resolve payload paths if they are files. - for name, payload := range r.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - r.Payloads[name] = final - } - } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) - if err != nil { - return errors.Wrap(err, "could not parse payloads") - } - } - - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators - if err := compiled.Compile(); err != nil { - return errors.Wrap(err, "could not compile operators") - } - r.CompiledOperators = compiled - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (r *Request) Requests() int { - if r.generator != nil { - return r.generator.NewIterator().Total() - } - return 1 -} - -// GetID returns the ID for the request if any. -func (r *Request) GetID() string { - return "" -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - hostname, err := getAddress(input) - if err != nil { - return nil - } - - if r.generator != nil { - iterator := r.generator.NewIterator() - - for { - value, ok := iterator.Value() - if !ok { - break - } - if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { - return err - } - } - } else { - value := make(map[string]interface{}) - if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { - return err - } - } - return nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - header := http.Header{} - - payloadValues := make(map[string]interface{}) - for k, v := range dynamicValues { - payloadValues[k] = v - } - parsed, err := url.Parse(input) - if err != nil { - return errors.Wrap(err, "could not parse input url") - } - payloadValues["Hostname"] = parsed.Host - payloadValues["Host"] = parsed.Hostname() - payloadValues["Scheme"] = parsed.Scheme - payloadValues["Path"] = parsed.Path - - for key, value := range r.Headers { - finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - header.Set(key, string(finalData)) - } - websocketDialer := ws.Dialer{ - Header: ws.HandshakeHeaderHTTP(header), - Timeout: time.Duration(r.options.Options.Timeout) * time.Second, - NetDial: r.dialer.Dial, - TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, - } - - finalAddress, dataErr := expressions.EvaluateByte([]byte(r.Address), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - addressToDial := string(finalAddress) - if parsed.Path != "" && parsed.Path != "/" { - addressToDial = addressToDial + parsed.Path - } - - conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not connect to server") - } - defer conn.Close() - - responseBuilder := &strings.Builder{} - if readBuffer != nil { - _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response - } - - reqBuilder := &strings.Builder{} - - inputEvents := make(map[string]interface{}) - for _, req := range r.Inputs { - reqBuilder.Grow(len(req.Data)) - - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) - if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - reqBuilder.WriteString(string(finalData)) - - err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - msg, _, err := wsutil.ReadServerData(conn) - if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - r.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - responseBuilder.Write(msg) - if req.Name != "" { - bufferStr := string(msg) - if req.Name != "" { - inputEvents[req.Name] = bufferStr - } - - // Run any internal extractors for the request here and add found values to map. - if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) - for k, v := range values { - dynamicValues[k] = v - } - } - } - } - r.options.Progress.IncrementRequests() - - if r.options.Options.Debug || r.options.Options.DebugRequests { - requestOutput := reqBuilder.String() - gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", requestOutput) - } - - r.options.Output.Request(r.options.TemplateID, input, "websocket", err) - gologger.Verbose().Msgf("Sent Websocket request to %s", input) - - data := make(map[string]interface{}) - for k, v := range previous { - data[k] = v - } - for k, v := range dynamicValues { - data[k] = v - } - for k, v := range inputEvents { - data[k] = v - } - data["success"] = "true" - data["request"] = reqBuilder.String() - data["response"] = responseBuilder.String() - data["host"] = input - data["ip"] = r.dialer.GetDialedIP(hostname) - - event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues - }) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) - } - - callback(event) - return nil -} - -// getAddress returns the address of the host to make request to -func getAddress(toTest string) (string, error) { - if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { - return "", errors.New("invalid websocket provided") - } - parsed, _ := url.Parse(toTest) - if parsed != nil && parsed.Host != "" { - return parsed.Host, nil - } - return "", nil -} - -// Match performs matching operation for a matcher on model and returns: -// true and a list of matched snippets if the matcher type is supports it -// otherwise false and an empty string slice -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - return protocols.MakeDefaultMatchFunc(data, matcher) -} - -// Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { - return protocols.MakeDefaultExtractFunc(data, matcher) -} - -// MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r, wrapped) -} - -// GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - return []*operators.Operators{r.CompiledOperators} -} - -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { - data := &output.ResultEvent{ - TemplateID: types.ToString(r.options.TemplateID), - TemplatePath: types.ToString(r.options.TemplatePath), - Info: r.options.TemplateInfo, - Type: "websocket", - Host: types.ToString(wrapped.InternalEvent["host"]), - Matched: types.ToString(wrapped.InternalEvent["host"]), - Metadata: wrapped.OperatorsResult.PayloadValues, - ExtractedResults: wrapped.OperatorsResult.OutputExtracts, - Timestamp: time.Now(), - IP: types.ToString(wrapped.InternalEvent["ip"]), - Request: types.ToString(wrapped.InternalEvent["request"]), - Response: types.ToString(wrapped.InternalEvent["response"]), - } - return data -} diff --git a/v2/pkg/protocols/others/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go similarity index 65% rename from v2/pkg/protocols/others/ssl/ssl.go rename to v2/pkg/protocols/ssl/ssl.go index f8f34b00b..3e9eb1174 100644 --- a/v2/pkg/protocols/others/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -36,37 +36,37 @@ type Request struct { } // Compile compiles the request generators preparing any requests possible. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - r.options = options +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + request.options = options client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) if err != nil { return errors.Wrap(err, "could not get network client") } - r.dialer = client + request.dialer = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } return nil } // Requests returns the total number of requests the rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } // GetID returns the ID for the request if any. -func (r *Request) GetID() string { +func (request *Request) GetID() string { return "" } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { return nil @@ -75,24 +75,24 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} - conn, err := r.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "ssl", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "ssl", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) connTLS, ok := conn.(*tls.Conn) if !ok { return nil } - r.options.Output.Request(r.options.TemplateID, address, "ssl", err) + request.options.Output.Request(request.options.TemplateID, address, "ssl", err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", r.options.TemplateID, input) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Str("address", input).Msgf("[%s] Dumped SSL request for %s", request.options.TemplateID, input) } state := connTLS.ConnectionState() @@ -110,13 +110,13 @@ func (r *Request) ExecuteWithResults(input string, dynamicValues, previous outpu data["response"] = jsonDataString data["host"] = input data["not_after"] = float64(cert.NotAfter.Unix()) - data["ip"] = r.dialer.GetDialedIP(hostname) + data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEvent(r, data, r.options.Options.Debug || r.options.Options.DebugResponse) - if r.options.Options.Debug || r.options.Options.DebugResponse { + event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) + if request.options.Options.Debug || request.options.Options.DebugResponse { responseOutput := jsonDataString - gologger.Debug().Msgf("[%s] Dumped SSL response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor)) } callback(event) return nil @@ -144,30 +144,30 @@ func getAddress(toTest string) (string, error) { // Match performs matching operation for a matcher on model and returns: // true and a list of matched snippets if the matcher type is supports it // otherwise false and an empty string slice -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { return protocols.MakeDefaultMatchFunc(data, matcher) } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { return protocols.MakeDefaultExtractFunc(data, matcher) } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r, wrapped) +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } // GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - return []*operators.Operators{r.CompiledOperators} +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} } -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: types.ToString(r.options.TemplateID), - TemplatePath: types.ToString(r.options.TemplatePath), - Info: r.options.TemplateInfo, + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, Type: "ssl", Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/others/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go similarity index 100% rename from v2/pkg/protocols/others/ssl/ssl_test.go rename to v2/pkg/protocols/ssl/ssl_test.go diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 79e6d766d..7e7f05a9d 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,8 +9,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/ssl" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) From 47949c0b522ce305aa50a924273809a468acd8d5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 17:18:35 +0530 Subject: [PATCH 079/196] Use separate type enum for protocol types --- v2/cmd/nuclei/main.go | 5 +- v2/internal/runner/runner.go | 4 +- v2/pkg/catalog/loader/filter/tag_filter.go | 26 +-- .../catalog/loader/filter/tag_filter_test.go | 43 ++--- v2/pkg/catalog/loader/loader.go | 9 +- v2/pkg/parsers/parser.go | 3 +- v2/pkg/templates/templates.go | 17 +- v2/pkg/templates/types/types.go | 154 ++++++++++++++++++ v2/pkg/types/types.go | 5 +- 9 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 v2/pkg/templates/types/types.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef65582..3d6570240 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -68,8 +69,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.VarP(&options.Protocols, "type", "tp", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), + flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "etype", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c4c550ea4..78ed7f132 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,8 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, - Types: r.options.Protocols, - ExcludeTypes: r.options.ExcludeProtocols, + Protocols: r.options.Protocols, + ExcludeProtocols: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index c33e96fd7..1420003ec 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // TagFilter is used to filter nuclei templates for tag based execution @@ -15,8 +16,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} - types map[string]struct{} - excludeTypes map[string]struct{} + types map[types.ProtocolType]struct{} + excludeTypes map[types.ProtocolType]struct{} } // ErrExcluded is returned for excluded templates @@ -27,7 +28,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -121,8 +122,11 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } -func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { - if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { +func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType) bool { + if len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0 { + return true + } + if templateType.String() == "" || templateType == types.InvalidProtocol { return true } @@ -146,8 +150,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes } // New returns a tag filter for nuclei tag based execution @@ -161,8 +165,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), - types: make(map[string]struct{}), - excludeTypes: make(map[string]struct{}), + types: make(map[types.ProtocolType]struct{}), + excludeTypes: make(map[types.ProtocolType]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -204,12 +208,12 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } - for _, tag := range config.Types { + for _, tag := range config.Protocols { if _, ok := filter.types[tag]; !ok { filter.types[tag] = struct{}{} } } - for _, tag := range config.ExcludeTypes { + for _, tag := range config.ExcludeProtocols { if _, ok := filter.excludeTypes[tag]; !ok { filter.excludeTypes[tag] = struct{}{} } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 7599c27b0..0758f6d4a 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) func TestTagBasedFilter(t *testing.T) { @@ -15,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } @@ -36,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +64,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +89,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,33 +98,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-type", func(t *testing.T) { filter := New(&Config{ - Types: []string{"http"}, + Protocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-type", func(t *testing.T) { filter := New(&Config{ - ExcludeTypes: []string{"http"}, + ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index fe725e2ee..604a7da51 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Config contains the configuration options for the loader @@ -21,8 +22,8 @@ type Config struct { Tags []string ExcludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -58,8 +59,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, - Types: config.Types, - ExcludeTypes: config.ExcludeTypes, + Protocols: config.Protocols, + ExcludeProtocols: config.ExcludeProtocols, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 9e201fb08..e26e7488e 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) @@ -59,7 +60,7 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 1e84def62..ff9e527f4 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -75,19 +76,21 @@ type Template struct { } // Type returns the type of the template -func (t *Template) Type() string { +func (t *Template) Type() types.ProtocolType { switch { case len(t.RequestsDNS) > 0: - return "dns" + return types.DNSProtocol case len(t.RequestsFile) > 0: - return "file" + return types.FileProtocol case len(t.RequestsHTTP) > 0: - return "http" + return types.HTTPProtocol case len(t.RequestsHeadless) > 0: - return "headless" + return types.HeadlessProtocol case len(t.RequestsNetwork) > 0: - return "network" + return types.NetworkProtocol + case t.CompiledWorkflow != nil: + return types.WorkflowProtocol default: - return "" + return types.InvalidProtocol } } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go new file mode 100644 index 000000000..f82bac5b1 --- /dev/null +++ b/v2/pkg/templates/types/types.go @@ -0,0 +1,154 @@ +package types + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" +) + +// ProtocolType is the type of the request protocol specified +type ProtocolType int + +// Supported values for the ProtocolType +const ( + DNSProtocol ProtocolType = iota + 1 + FileProtocol + HTTPProtocol + HeadlessProtocol + NetworkProtocol + WorkflowProtocol + limit + InvalidProtocol +) + +// ExtractorTypes is a table for conversion of extractor type from string. +var protocolMappings = map[ProtocolType]string{ + InvalidProtocol: "invalid", + DNSProtocol: "dns", + FileProtocol: "file", + HTTPProtocol: "http", + HeadlessProtocol: "headless", + NetworkProtocol: "network", + WorkflowProtocol: "workflow", +} + +func GetSupportedProtocolTypes() ProtocolTypes { + var result []ProtocolType + for index := ProtocolType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toProtocolType(valueToMap string) (ProtocolType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range protocolMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid protocol type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ProtocolType) String() string { + return protocolMappings[t] +} + +// TypeHolder is used to hold internal type of the protocol +type TypeHolder struct { + ProtocolType ProtocolType +} + +func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the protocol", + Description: "Type of the protocol", + } + for _, types := range GetSupportedProtocolTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toProtocolType(marshalledTypes) + if err != nil { + return err + } + + holder.ProtocolType = computedType + return nil +} + +func (holder *TypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ProtocolType.String()) +} + +func (holder TypeHolder) MarshalYAML() (interface{}, error) { + return holder.ProtocolType.String(), nil +} + +type ProtocolTypes []ProtocolType + +func (protocolTypes *ProtocolTypes) Set(values string) error { + inputTypes, err := goflags.ToNormalizedStringSlice(values) + if err != nil { + return err + } + + for _, inputType := range inputTypes { + if err := setProtocolType(protocolTypes, inputType); err != nil { + return err + } + } + return nil +} + +func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringSliceValue stringslice.StringSlice + if err := unmarshal(&stringSliceValue); err != nil { + return err + } + + stringSLice := stringSliceValue.ToSlice() + var result = make(ProtocolTypes, 0, len(stringSLice)) + for _, typeString := range stringSLice { + if err := setProtocolType(&result, typeString); err != nil { + return err + } + } + *protocolTypes = result + return nil +} + +func (protocolTypes ProtocolTypes) String() string { + var stringTypes []string + for _, t := range protocolTypes { + stringTypes = append(stringTypes, t.String()) + } + return strings.Join(stringTypes, ", ") +} + +func setProtocolType(protocolTypes *ProtocolTypes, value string) error { + computedType, err := toProtocolType(value) + if err != nil { + return fmt.Errorf("'%s' is not a valid extract type", value) + } + *protocolTypes = append(*protocolTypes, computedType) + return nil +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 36f66503e..fc65e3a1f 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -3,6 +3,7 @@ package types import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Options contains the configuration options for nuclei scanner. @@ -30,9 +31,9 @@ type Options struct { // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities // Protocols contains the protocols to be allowed executed - Protocols goflags.NormalizedStringSlice + Protocols types.ProtocolTypes // ExcludeProtocols contains protocols to not be executed - ExcludeProtocols goflags.NormalizedStringSlice + ExcludeProtocols types.ProtocolTypes // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From 4a0229c13d706860cd01fd1d7623762ea15e22b2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 18:36:54 +0530 Subject: [PATCH 080/196] Websocket protocol rename --- v2/pkg/protocols/websocket/websocket.go | 365 ++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 v2/pkg/protocols/websocket/websocket.go diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go new file mode 100644 index 000000000..6aa90ede3 --- /dev/null +++ b/v2/pkg/protocols/websocket/websocket.go @@ -0,0 +1,365 @@ +package websocket + +import ( + "context" + "crypto/tls" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` + // description: | + // Inputs contains inputs for the websocket protocol + Inputs []*Input `yaml:"inputs,omitempty" jsonschema:"title=inputs for the websocket request,description=Inputs contains any input/output for the current request"` + // description: | + // Headers contains headers for the request. + Headers map[string]string `yaml:"headers,omitempty" jsonschema:"title=headers contains the request headers,description=Headers contains headers for the request"` + + // description: | + // Attack is the type of payload combinations to perform. + // + // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // permutations and combinations for all payloads. + // values: + // - "sniper" + // - "pitchfork" + // - "clusterbomb" + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + // description: | + // Payloads contains any payloads for the current request. + // + // Payloads support both key-values combinations where a list + // of payloads is provided, or optionally a single file can also + // be provided as payload which will be read on run-time. + Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` + + generator *generators.Generator + attackType generators.Type + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Input is an input for the websocket protocol +type Input struct { + // description: | + // Data is the data to send as the input. + // + // It supports DSL Helper Functions as well as normal expressions. + // examples: + // - value: "\"TEST\"" + // - value: "\"hex_decode('50494e47')\"" + Data string `yaml:"data,omitempty" jsonschema:"title=data to send as input,description=Data is the data to send as the input"` + // description: | + // Name is the optional name of the data read to provide matching on. + // examples: + // - value: "\"prefix\"" + Name string `yaml:"name,omitempty" jsonschema:"title=optional name for data read,description=Optional name of the data read to provide matching on"` +} + +// Compile compiles the request generators preparing any requests possible. +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + request.options = options + + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + request.dialer = client + + if len(request.Payloads) > 0 { + attackType := request.AttackType + if attackType == "" { + attackType = "batteringram" + } + request.attackType = generators.StringToType[attackType] + + // Resolve payload paths if they are files. + for name, payload := range request.Payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) + if resolveErr != nil { + return errors.Wrap(resolveErr, "could not read payload file") + } + request.Payloads[name] = final + } + } + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + if err != nil { + return errors.Wrap(err, "could not parse payloads") + } + } + + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + request.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (request *Request) Requests() int { + if request.generator != nil { + return request.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (request *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + hostname, err := getAddress(input) + if err != nil { + return nil + } + + if request.generator != nil { + iterator := request.generator.NewIterator() + + for { + value, ok := iterator.Value() + if !ok { + break + } + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + } else { + value := make(map[string]interface{}) + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + header := http.Header{} + + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + parsed, err := url.Parse(input) + if err != nil { + return errors.Wrap(err, "could not parse input url") + } + payloadValues["Hostname"] = parsed.Host + payloadValues["Host"] = parsed.Hostname() + payloadValues["Scheme"] = parsed.Scheme + payloadValues["Path"] = parsed.Path + + for key, value := range request.Headers { + finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + header.Set(key, string(finalData)) + } + websocketDialer := ws.Dialer{ + Header: ws.HandshakeHeaderHTTP(header), + Timeout: time.Duration(request.options.Options.Timeout) * time.Second, + NetDial: request.dialer.Dial, + TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, + } + + finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + addressToDial := string(finalAddress) + if parsed.Path != "" && parsed.Path != "/" { + addressToDial = addressToDial + parsed.Path + } + + conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not connect to server") + } + defer conn.Close() + + responseBuilder := &strings.Builder{} + if readBuffer != nil { + _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response + } + + reqBuilder := &strings.Builder{} + + inputEvents := make(map[string]interface{}) + for _, req := range request.Inputs { + reqBuilder.Grow(len(req.Data)) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + reqBuilder.WriteString(string(finalData)) + + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + msg, _, err := wsutil.ReadServerData(conn) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not write request to server") + } + + responseBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + if req.Name != "" { + inputEvents[req.Name] = bufferStr + } + + // Run any internal extractors for the request here and add found values to map. + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) + for k, v := range values { + dynamicValues[k] = v + } + } + } + } + request.options.Progress.IncrementRequests() + + if request.options.Options.Debug || request.options.Options.DebugRequests { + requestOutput := reqBuilder.String() + gologger.Info().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", requestOutput) + } + + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + gologger.Verbose().Msgf("Sent Websocket request to %s", input) + + data := make(map[string]interface{}) + for k, v := range previous { + data[k] = v + } + for k, v := range dynamicValues { + data[k] = v + } + for k, v := range inputEvents { + data[k] = v + } + data["success"] = "true" + data["request"] = reqBuilder.String() + data["response"] = responseBuilder.String() + data["host"] = input + data["ip"] = request.dialer.GetDialedIP(hostname) + + event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues + }) + if request.options.Options.Debug || request.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor)) + } + + callback(event) + return nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if !strings.HasPrefix(toTest, "ws://") && !strings.HasPrefix(toTest, "wss://") { + return "", errors.New("invalid websocket provided") + } + parsed, _ := url.Parse(toTest) + if parsed != nil && parsed.Host != "" { + return parsed.Host, nil + } + return "", nil +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, + Type: "websocket", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["response"]), + } + return data +} From 645ae30a47b998f2ca17d69ec2bc75f594684bcd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 19:53:45 +0530 Subject: [PATCH 081/196] Moved to an enum for TemplateType in protocols --- v2/pkg/protocols/dns/dns.go | 6 +-- v2/pkg/protocols/dns/dns_test.go | 12 +++--- v2/pkg/protocols/dns/operators_test.go | 48 +++++++++++----------- v2/pkg/protocols/dns/request.go | 6 +++ v2/pkg/protocols/dns/request_test.go | 12 +++--- v2/pkg/protocols/file/request.go | 6 +++ v2/pkg/protocols/headless/request.go | 6 +++ v2/pkg/protocols/http/request.go | 6 +++ v2/pkg/protocols/network/request.go | 6 +++ v2/pkg/protocols/offlinehttp/request.go | 6 +++ v2/pkg/protocols/ssl/ssl.go | 8 +++- v2/pkg/protocols/websocket/websocket.go | 8 +++- v2/pkg/templates/templates_doc_examples.go | 10 ++--- 13 files changed, 94 insertions(+), 46 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 8aab88af2..a42df7ef1 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -32,7 +32,7 @@ type Request struct { // - value: "\"{{FQDN}}\"" Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"` // description: | - // Type is the type of DNS request to make. + // RequestType is the type of DNS request to make. // values: // - "A" // - "NS" @@ -43,7 +43,7 @@ type Request struct { // - "MX" // - "TXT" // - "AAAA" - Type string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` + RequestType string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. // @@ -111,7 +111,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } request.class = classToInt(request.Class) request.options = options - request.question = questionTypeToInt(request.Type) + request.question = questionTypeToInt(request.RequestType) return nil } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index 85b00bd94..d01f84bdc 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -27,12 +27,12 @@ func TestDNSCompileMake(t *testing.T) { testutils.Init(options) const templateID = "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 6bb91b7e2..0e4c6ff3e 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -23,12 +23,12 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -55,12 +55,12 @@ func TestDNSOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -166,12 +166,12 @@ func TestDNSOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -223,12 +223,12 @@ func TestDNSMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d063825f6..05d4930da 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,10 +12,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.DNSProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index c77bffaed..bc5f230b3 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -20,12 +20,12 @@ func TestDNSExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - Type: "A", - Class: "INET", - Retries: 5, - ID: templateID, - Recursion: false, - Name: "{{FQDN}}", + RequestType: "A", + Class: "INET", + Retries: 5, + ID: templateID, + Recursion: false, + Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index c7c3daee9..ef650bab8 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -15,10 +15,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.FileProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 1609a595b..dc1d3467c 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -12,10 +12,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HeadlessProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf5..af5075f39 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -28,12 +28,18 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" ) const defaultMaxWorkers = 150 +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HTTPProtocol +} + // executeRaceRequest executes race condition request for a URL func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var generatedRequests []*generatedRequest diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 761350856..ed52117fb 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -21,10 +21,16 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.NetworkProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var address string diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index c9dbc1a82..10b77b599 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -15,12 +15,18 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.HTTPProtocol +} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index ec79f332a..6335e879b 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -21,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -163,12 +164,17 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.SSLProtocol +} + func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: "ssl", + Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index f17a3b11a..076855dab 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -25,6 +25,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -355,7 +356,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: "websocket", + Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, @@ -367,3 +368,8 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent } return data } + +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.WebsocketProtocol +} diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 84580ffe4..b755de290 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -37,11 +37,11 @@ var ( _ = exampleNormalHTTPRequest exampleNormalDNSRequest = &dns.Request{ - Name: "{{FQDN}}", - Type: "CNAME", - Class: "inet", - Retries: 2, - Recursion: true, + Name: "{{FQDN}}", + RequestType: "CNAME", + Class: "inet", + Retries: 2, + Recursion: true, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ {Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, From 91a7b4df6aa2e71e7d17c7d2642a2979d8cf980d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 20:08:11 +0530 Subject: [PATCH 082/196] Validate opcode before proceeding with websocket --- v2/pkg/protocols/websocket/websocket.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 076855dab..1ae6bbaa9 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -251,12 +251,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam return errors.Wrap(err, "could not write request to server") } - msg, _, err := wsutil.ReadServerData(conn) + msg, opCode, err := wsutil.ReadServerData(conn) if err != nil { request.options.Output.Request(request.options.TemplateID, input, "websocket", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } + // Only perform matching and writes in case we recieve + // text or binary opcode from the websocket server. + if opCode != ws.OpText && opCode != ws.OpBinary { + continue + } responseBuilder.Write(msg) if req.Name != "" { From a7c8d0473c1f94692d4d69b2d8b7611db3db8790 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:28:48 +0530 Subject: [PATCH 083/196] Removed duplicate attackType logic + move attackType to generators --- .../protocols/common/generators/generators.go | 39 ++++++++++++++++--- .../common/generators/generators_test.go | 10 +++-- v2/pkg/protocols/http/http.go | 24 +----------- .../protocols/http/request_generator_test.go | 11 ++++-- v2/pkg/protocols/network/network.go | 27 +------------ v2/pkg/protocols/websocket/websocket.go | 22 +---------- 6 files changed, 53 insertions(+), 80 deletions(-) diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index a63fbcb2b..15ac15131 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -2,7 +2,12 @@ package generators -import "github.com/pkg/errors" +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" +) // Generator is the generator struct for generating payloads type Generator struct { @@ -30,21 +35,45 @@ var StringToType = map[string]Type{ } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, payloadType Type, templatePath string) (*Generator, error) { +func New(payloads map[string]interface{}, attackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { + if attackType == "" { + attackType = "batteringram" + } + attackTypeValue, ok := StringToType[attackType] + if !ok { + return nil, fmt.Errorf("invalid attack type provided: %s", attackType) + } + + // Resolve payload paths if they are files. + payloadsFinal := make(map[string]interface{}) + for name, payload := range payloads { + payloadsFinal[name] = payload + } + for name, payload := range payloads { + payloadStr, ok := payload.(string) + if ok { + final, resolveErr := catalog.ResolvePath(payloadStr, templatePath) + if resolveErr != nil { + return nil, errors.Wrap(resolveErr, "could not read payload file") + } + payloadsFinal[name] = final + } + } + generator := &Generator{} if err := generator.validate(payloads, templatePath); err != nil { return nil, err } - compiled, err := loadPayloads(payloads) + compiled, err := loadPayloads(payloadsFinal) if err != nil { return nil, err } - generator.Type = payloadType + generator.Type = attackTypeValue generator.payloads = compiled // Validate the batteringram payload set - if payloadType == BatteringRam { + if attackTypeValue == BatteringRam { if len(payloads) != 1 { return nil, errors.New("batteringram must have single payload set") } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index de37ca62d..8f6d9893e 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -3,13 +3,15 @@ package generators import ( "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/stretchr/testify/require" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames}, "batteringram", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -28,7 +30,8 @@ func TestPitchforkGenerator(t *testing.T) { usernames := []string{"admin", "token"} passwords := []string{"password1", "password2", "password3"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "pitchfork", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -49,7 +52,8 @@ func TestClusterbombGenerator(t *testing.T) { usernames := []string{"admin"} passwords := []string{"admin", "password", "token"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "") + catalogInstance := catalog.New("") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "clusterbomb", "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 304f1a497..1c71ea5f1 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -129,7 +129,6 @@ type Request struct { CompiledOperators *operators.Operators `yaml:"-"` options *protocols.ExecuterOptions - attackType generators.Type totalRequests int customHeaders map[string]string generator *generators.Generator // optional, only enabled when using payloads @@ -267,28 +266,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - var ok bool - request.attackType, ok = generators.StringToType[attackType] - if !ok { - return fmt.Errorf("invalid attack type provided: %s", attackType) - } - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 53064cca0..c6ebec683 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -3,6 +3,7 @@ package http import ( "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/stretchr/testify/require" ) @@ -28,10 +29,11 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - attackType: generators.ClusterBomb, + AttackType: "clusterbomb", Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType, "") + catalogInstance := catalog.New("") + req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -51,10 +53,11 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - attackType: generators.ClusterBomb, + AttackType: "clusterbomb", Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType, "") + catalogInstance := catalog.New("") + req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 12094ae6e..c0a0fa2e0 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,7 +1,6 @@ package network import ( - "fmt" "net" "strings" @@ -76,8 +75,7 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` - generator *generators.Generator - attackType generators.Type + generator *generators.Generator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer options *protocols.ExecuterOptions @@ -186,28 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - var ok bool - request.attackType, ok = generators.StringToType[attackType] - if !ok { - return fmt.Errorf("invalid attack type provided: %s", attackType) - } - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 1ae6bbaa9..ffe6a0d20 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -63,8 +63,7 @@ type Request struct { // be provided as payload which will be read on run-time. Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - generator *generators.Generator - attackType generators.Type + generator *generators.Generator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -99,24 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - attackType := request.AttackType - if attackType == "" { - attackType = "batteringram" - } - request.attackType = generators.StringToType[attackType] - - // Resolve payload paths if they are files. - for name, payload := range request.Payloads { - payloadStr, ok := payload.(string) - if ok { - final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) - if resolveErr != nil { - return errors.Wrap(resolveErr, "could not read payload file") - } - request.Payloads[name] = final - } - } - request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } From a60b10afca45300759c3be636e4e0e9b5f487bff Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:41:56 +0530 Subject: [PATCH 084/196] Added an enum for attackType --- .../common/generators/attack_types.go | 96 +++++++++++++++++++ .../protocols/common/generators/generators.go | 51 +++------- .../common/generators/generators_test.go | 6 +- v2/pkg/protocols/http/build_request_test.go | 5 +- v2/pkg/protocols/http/http.go | 4 +- v2/pkg/protocols/http/http_test.go | 3 +- .../protocols/http/request_generator_test.go | 8 +- v2/pkg/protocols/network/network.go | 4 +- v2/pkg/protocols/websocket/websocket.go | 4 +- 9 files changed, 127 insertions(+), 54 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/attack_types.go diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go new file mode 100644 index 000000000..ea445cbc6 --- /dev/null +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -0,0 +1,96 @@ +package generators + +import ( + "encoding/json" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" +) + +// AttackType is the type of attack for payloads +type AttackType int + +// Supported values for the ProtocolType +const ( + // BatteringRamAttack replaces same payload into all of the defined payload positions at once. + BatteringRamAttack AttackType = iota + 1 + // PitchForkAttack replaces variables with positional value from multiple wordlists + PitchForkAttack + // ClusterbombAttack replaces variables with all possible combinations of values + ClusterbombAttack + limit +) + +// attackTypeMappings is a table for conversion of attack type from string. +var attackTypeMappings = map[AttackType]string{ + BatteringRamAttack: "batteringram", + PitchForkAttack: "pitchfork", + ClusterbombAttack: "clusterbomb", +} + +func GetSupportedAttackTypes() []AttackType { + var result []AttackType + for index := AttackType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toAttackType(valueToMap string) (AttackType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range attackTypeMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("invalid attack type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t AttackType) String() string { + return attackTypeMappings[t] +} + +// AttackTypeHolder is used to hold internal type of the protocol +type AttackTypeHolder struct { + Value AttackType +} + +func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the protocol", + Description: "Type of the protocol", + } + for _, types := range GetSupportedAttackTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *AttackTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toAttackType(marshalledTypes) + if err != nil { + return err + } + + holder.Value = computedType + return nil +} + +func (holder *AttackTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.Value.String()) +} + +func (holder AttackTypeHolder) MarshalYAML() (interface{}, error) { + return holder.Value.String(), nil +} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 15ac15131..47b2fa44a 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -3,45 +3,20 @@ package generators import ( - "fmt" - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) // Generator is the generator struct for generating payloads type Generator struct { - Type Type + Type AttackType payloads map[string][]string } -// Type is type of attack -type Type int - -const ( - // Batteringram replaces same payload into all of the defined payload positions at once. - BatteringRam Type = iota + 1 - // PitchFork replaces variables with positional value from multiple wordlists - PitchFork - // ClusterBomb replaces variables with all possible combinations of values - ClusterBomb -) - -// StringToType is a table for conversion of attack type from string. -var StringToType = map[string]Type{ - "batteringram": BatteringRam, - "pitchfork": PitchFork, - "clusterbomb": ClusterBomb, -} - // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { - if attackType == "" { - attackType = "batteringram" - } - attackTypeValue, ok := StringToType[attackType] - if !ok { - return nil, fmt.Errorf("invalid attack type provided: %s", attackType) +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { + if attackType.String() == "" { + attackType = BatteringRamAttack } // Resolve payload paths if they are files. @@ -69,11 +44,11 @@ func New(payloads map[string]interface{}, attackType, templatePath string, catal if err != nil { return nil, err } - generator.Type = attackTypeValue + generator.Type = attackType generator.payloads = compiled // Validate the batteringram payload set - if attackTypeValue == BatteringRam { + if attackType == BatteringRamAttack { if len(payloads) != 1 { return nil, errors.New("batteringram must have single payload set") } @@ -83,7 +58,7 @@ func New(payloads map[string]interface{}, attackType, templatePath string, catal // Iterator is a single instance of an iterator for a generator structure type Iterator struct { - Type Type + Type AttackType position int msbIterator int total int @@ -124,18 +99,18 @@ func (i *Iterator) Remaining() int { func (i *Iterator) Total() int { count := 0 switch i.Type { - case BatteringRam: + case BatteringRamAttack: for _, p := range i.payloads { count += len(p.values) } - case PitchFork: + case PitchForkAttack: count = len(i.payloads[0].values) for _, p := range i.payloads { if count > len(p.values) { count = len(p.values) } } - case ClusterBomb: + case ClusterbombAttack: count = 1 for _, p := range i.payloads { count *= len(p.values) @@ -147,11 +122,11 @@ func (i *Iterator) Total() int { // Value returns the next value for an iterator func (i *Iterator) Value() (map[string]interface{}, bool) { switch i.Type { - case BatteringRam: + case BatteringRamAttack: return i.batteringRamValue() - case PitchFork: + case PitchForkAttack: return i.pitchforkValue() - case ClusterBomb: + case ClusterbombAttack: return i.clusterbombValue() default: return i.batteringRamValue() diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 8f6d9893e..75e39d0ed 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -11,7 +11,7 @@ func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames}, "batteringram", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -31,7 +31,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "pitchfork", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -53,7 +53,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, "clusterbomb", "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterbombAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index d45f7f8f2..8443a7f6e 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { @@ -133,7 +134,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -172,7 +173,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 1c71ea5f1..df79e8f47 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -52,7 +52,7 @@ type Request struct { // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. // values: @@ -266,7 +266,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 5f0e304b8..b7da9da67 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) func TestHTTPCompile(t *testing.T) { @@ -22,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index c6ebec683..272a62829 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -29,11 +29,11 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") - req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -53,11 +53,11 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: "clusterbomb", + AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") - req.generator, err = generators.New(req.Payloads, req.AttackType, "", catalogInstance) + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance) require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c0a0fa2e0..6b3cbe2a6 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -40,7 +40,7 @@ type Request struct { // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, request.options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index ffe6a0d20..8079c777f 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -54,7 +54,7 @@ type Request struct { // - "sniper" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -98,7 +98,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType, request.options.TemplatePath, options.Catalog) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog) if err != nil { return errors.Wrap(err, "could not parse payloads") } From 53164c3f3e1cdd4af471f1e813b5c2d4129c9a5b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:44:47 +0530 Subject: [PATCH 085/196] Misc linter related changes --- v2/pkg/protocols/common/generators/attack_types.go | 4 ++-- v2/pkg/protocols/http/build_request_test.go | 4 ++-- v2/pkg/protocols/http/http_test.go | 2 +- v2/pkg/protocols/http/request_generator_test.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index ea445cbc6..769026907 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -63,8 +63,8 @@ type AttackTypeHolder struct { func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { gotType := &jsonschema.Type{ Type: "string", - Title: "type of the protocol", - Description: "Type of the protocol", + Title: "type of the attack", + Description: "Type of the attack", } for _, types := range GetSupportedAttackTypes() { gotType.Enum = append(gotType.Enum, types.String()) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 8443a7f6e..c5cc3aaff 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -134,7 +134,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -173,7 +173,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index b7da9da67..69a71fb29 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -23,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 272a62829..3fee8024c 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -29,7 +29,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") @@ -53,7 +53,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") From 12321c23af3be5e199202066f5cf930f46a012c4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 02:56:59 +0530 Subject: [PATCH 086/196] Splitting big function into small --- v2/pkg/protocols/ssl/ssl.go | 3 +- v2/pkg/protocols/websocket/websocket.go | 113 +++++++++++++----------- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 6335e879b..8f624d5b7 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -115,9 +115,8 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { - responseOutput := jsonDataString gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor, false)) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) } callback(event) return nil diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 8079c777f..4cea98d59 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "io" + "net" "net/http" "net/url" "strings" @@ -212,57 +213,15 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam _, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response } - reqBuilder := &strings.Builder{} - - inputEvents := make(map[string]interface{}) - for _, req := range request.Inputs { - reqBuilder.Grow(len(req.Data)) - - finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) - if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not evaluate template expressions") - } - reqBuilder.WriteString(string(finalData)) - - err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) - if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - - msg, opCode, err := wsutil.ReadServerData(conn) - if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(err, "could not write request to server") - } - // Only perform matching and writes in case we recieve - // text or binary opcode from the websocket server. - if opCode != ws.OpText && opCode != ws.OpBinary { - continue - } - - responseBuilder.Write(msg) - if req.Name != "" { - bufferStr := string(msg) - inputEvents[req.Name] = bufferStr - - // Run any internal extractors for the request here and add found values to map. - if request.CompiledOperators != nil { - values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) - for k, v := range values { - dynamicValues[k] = v - } - } - } + events, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(err, "could not read write response") } request.options.Progress.IncrementRequests() if request.options.Options.Debug || request.options.Options.DebugRequests { - requestOutput := reqBuilder.String() gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) gologger.Print().Msgf("%s", requestOutput) } @@ -274,20 +233,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam for k, v := range previous { data[k] = v } - for k, v := range dynamicValues { - data[k] = v - } - for k, v := range inputEvents { + for k, v := range events { data[k] = v } data["success"] = "true" - data["request"] = reqBuilder.String() + data["request"] = requestOutput data["response"] = responseBuilder.String() data["host"] = input data["ip"] = request.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = dynamicValues + internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) if request.options.Options.Debug || request.options.Options.DebugResponse { responseOutput := responseBuilder.String() @@ -299,6 +255,57 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam return nil } +func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map[string]interface{}, input string, respBuilder *strings.Builder) (events map[string]interface{}, req string, err error) { + reqBuilder := &strings.Builder{} + inputEvents := make(map[string]interface{}) + + for _, req := range request.Inputs { + reqBuilder.Grow(len(req.Data)) + + finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) + if dataErr != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions") + } + reqBuilder.WriteString(string(finalData)) + + err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(err, "could not write request to server") + } + + msg, opCode, err := wsutil.ReadServerData(conn) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil, "", errors.Wrap(err, "could not write request to server") + } + // Only perform matching and writes in case we recieve + // text or binary opcode from the websocket server. + if opCode != ws.OpText && opCode != ws.OpBinary { + continue + } + + respBuilder.Write(msg) + if req.Name != "" { + bufferStr := string(msg) + inputEvents[req.Name] = bufferStr + + // Run any internal extractors for the request here and add found values to map. + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc) + for k, v := range values { + inputEvents[k] = v + } + } + } + } + return inputEvents, reqBuilder.String(), nil +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { parsed, err := url.Parse(toTest) From f3675d547aeccfbdf9016b99e2cd422a4a2dd134 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 4 Nov 2021 03:02:29 +0530 Subject: [PATCH 087/196] URL parsing addition for websocket URLs --- v2/pkg/protocols/websocket/websocket.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 4cea98d59..7731dc29c 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/url" + "path" "strings" "time" @@ -195,9 +196,18 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } + addressToDial := string(finalAddress) + parsedAddress, err := url.Parse(addressToDial) + if err != nil { + request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not parse input url") + } + if parsed.Path != "" && parsed.Path != "/" { - addressToDial = addressToDial + parsed.Path + parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) + addressToDial = parsedAddress.String() } conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) @@ -240,6 +250,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["request"] = requestOutput data["response"] = responseBuilder.String() data["host"] = input + data["matched"] = addressToDial data["ip"] = request.dialer.GetDialedIP(hostname) event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { @@ -352,7 +363,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent Info: request.options.TemplateInfo, Type: request.Type().String(), Host: types.ToString(wrapped.InternalEvent["host"]), - Matched: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), From 41393fc09dc6e24171e8854deb6fcefb827475ad Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Thu, 4 Nov 2021 17:13:47 +0530 Subject: [PATCH 088/196] unique payload per interactsh placeholder #1068 --- .../protocols/common/interactsh/interactsh.go | 55 ++++++++++--------- v2/pkg/protocols/http/build_request.go | 55 ++++++++----------- v2/pkg/protocols/http/build_request_test.go | 14 ++--- v2/pkg/protocols/http/request.go | 16 ++---- v2/pkg/protocols/http/request_generator.go | 1 + v2/pkg/protocols/network/request.go | 14 ++--- 6 files changed, 71 insertions(+), 84 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5c2914ff5..5fff7ac50 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -206,12 +206,14 @@ func (c *Client) Close() bool { // // It accepts data to replace as well as the URL to replace placeholders // with generated uniquely for each request. -func (c *Client) ReplaceMarkers(data, interactshURL string) string { - if !strings.Contains(data, interactshURLMarker) { - return data +func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) { + + for strings.Contains(data, interactshURLMarker) { + url := c.URL() + interactshURLs = append(interactshURLs, url) + data = strings.Replace(data, interactshURLMarker, url, 1) } - replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data) - return replaced + return data, interactshURLs } // MakeResultEventFunc is a result making function for nuclei @@ -227,30 +229,33 @@ type RequestData struct { } // RequestEvent is the event for a network request sent by nuclei. -func (c *Client) RequestEvent(interactshURL string, data *RequestData) { - id := strings.TrimSuffix(interactshURL, c.dotHostname) +func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { + for _, interactshURL := range interactshURLs { + id := strings.TrimSuffix(interactshURL, c.dotHostname) - interaction := c.interactions.Get(id) - if interaction != nil { - // If we have previous interactions, get them and process them. - interactions, ok := interaction.Value().([]*server.Interaction) - if !ok { - c.requests.Set(id, data, c.eviction) - return - } - matched := false - for _, interaction := range interactions { - if c.processInteractionForRequest(interaction, data) { - matched = true - break + interaction := c.interactions.Get(id) + if interaction != nil { + // If we have previous interactions, get them and process them. + interactions, ok := interaction.Value().([]*server.Interaction) + if !ok { + c.requests.Set(id, data, c.eviction) + return } + matched := false + for _, interaction := range interactions { + if c.processInteractionForRequest(interaction, data) { + matched = true + break + } + } + if matched { + c.interactions.Delete(id) + } + } else { + c.requests.Set(id, data, c.eviction) } - if matched { - c.interactions.Delete(id) - } - } else { - c.requests.Set(id, data, c.eviction) } + } // HasMatchers returns true if an operator has interactsh part diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ea83aea72..20f3d4801 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -38,6 +38,7 @@ type generatedRequest struct { pipelinedClient *rawhttp.PipelineClient request *retryablehttp.Request dynamicValues map[string]interface{} + interactshURLs []string } func (g *generatedRequest) URL() string { @@ -52,9 +53,9 @@ func (g *generatedRequest) URL() string { // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. -func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { +func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) { if r.request.SelfContained { - return r.makeSelfContainedRequest(dynamicValues, interactURL) + return r.makeSelfContainedRequest(dynamicValues) } // We get the next payload for the request. data, payloads, ok := r.nextValue() @@ -63,12 +64,10 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) - for payloadName, payloadValue := range payloads { - payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL) - } + for payloadName, payloadValue := range payloads { + payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) } parsed, err := url.Parse(baseURL) @@ -98,12 +97,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. if isRawRequest { - return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads) } - return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads) } -func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { +func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) { // We get the next payload for the request. data, payloads, ok := r.nextValue() if !ok { @@ -136,13 +135,13 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int generators.BuildPayloadFromOptions(r.request.options.Options), ) - return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads) } values := generators.MergeMaps( dynamicValues, generators.BuildPayloadFromOptions(r.request.options.Options), ) - return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads) } // Total returns the total number of requests for the generator @@ -171,10 +170,8 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { } // MakeHTTPRequestFromModel creates a *http.Request from a request template -func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}, interactURL string) (*generatedRequest, error) { - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) - } +func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) // Combine the template payloads along with base // request values. @@ -198,18 +195,16 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st return nil, err } - request, err := r.fillRequest(req, finalValues, interactURL) + request, err := r.fillRequest(req, finalValues) if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request -func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}, interactURL string) (*generatedRequest, error) { - if interactURL != "" { - data = r.options.Interactsh.ReplaceMarkers(data, interactURL) - } +func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } @@ -258,21 +253,19 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest req.Host = value } } - request, err := r.fillRequest(req, finalValues, "") + request, err := r.fillRequest(req, finalValues) if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues, interactshURLs: r.interactshURLs}, nil } // fillRequest fills various headers in the request with values -func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}, interactURL string) (*retryablehttp.Request, error) { +func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { // Set the header values requested for header, value := range r.request.Headers { - if interactURL != "" { - value = r.options.Interactsh.ReplaceMarkers(value, interactURL) - } + value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) value, err := expressions.Evaluate(value, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") @@ -290,10 +283,8 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte // Check if the user requested a request body if r.request.Body != "" { - body := r.request.Body - if interactURL != "" { - body = r.options.Interactsh.ReplaceMarkers(body, interactURL) - } + var body string + body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) body, err := expressions.Evaluate(body, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index d45f7f8f2..122d81bce 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -84,7 +84,7 @@ func TestMakeRequestFromModal(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") bodyBytes, _ := req.request.BodyBytes() @@ -111,12 +111,12 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path") generator = request.newGenerator() - req, err = generator.Make("https://example.com/test/", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com/test/", map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path") } @@ -149,12 +149,12 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw") @@ -188,12 +188,12 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}, "") + req, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}, "") + req, err = generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b11340bf5..c3472b457 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -41,7 +41,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent generator := request.newGenerator() - requestForDump, err := generator.Make(reqURL, nil, "") + requestForDump, err := generator.Make(reqURL, nil) if err != nil { return err } @@ -59,7 +59,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Pre-Generate requests for i := 0; i < request.RaceNumberRequests; i++ { generator := request.newGenerator() - generatedRequest, err := generator.Make(reqURL, nil, "") + generatedRequest, err := generator.Make(reqURL, nil) if err != nil { return err } @@ -98,7 +98,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -161,7 +161,7 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -215,11 +215,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou for { hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) - var interactURL string - if request.options.Interactsh != nil && hasInteractMarkers { - interactURL = request.options.Interactsh.URL() - } - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL) + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { break } @@ -245,7 +241,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) } if hasInteractMarkers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 90c5d0fbd..783375835 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -16,6 +16,7 @@ type requestGenerator struct { request *Request options *protocols.ExecuterOptions payloadIterator *generators.Iterator + interactshURLs []string } // newGenerator creates a new request generator instance diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 761350856..7903ce4fa 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -120,11 +120,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input defer conn.Close() _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) - hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) - var interactURL string - if request.options.Interactsh != nil && hasInteractMarkers { - interactURL = request.options.Interactsh.URL() - } + var interactshURLs []string responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} @@ -137,9 +133,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input case "hex": data, err = hex.DecodeString(input.Data) default: - if interactURL != "" { - input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL) - } + input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{}) data = []byte(input.Data) } if err != nil { @@ -261,14 +255,14 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } var event *output.InternalWrappedEvent - if interactURL == "" { + if len(interactshURLs) == 0 { event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event) } else if request.options.Interactsh != nil { event = &output.InternalWrappedEvent{InternalEvent: outputEvent} - request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + request.options.Interactsh.RequestEvent(interactshURLs, &interactsh.RequestData{ MakeResultFunc: request.MakeResultEvent, Event: event, Operators: request.CompiledOperators, From 8ad3ebcd05be6adf03d6d4e11f439a450f4f0a35 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 03:01:41 +0530 Subject: [PATCH 089/196] Made code changes as per review comments --- v2/cmd/functional-test/main.go | 2 +- v2/cmd/integration-test/dns.go | 2 +- v2/cmd/integration-test/http.go | 2 +- v2/cmd/integration-test/integration-test.go | 2 +- v2/cmd/integration-test/loader.go | 5 +- v2/cmd/integration-test/network.go | 2 +- v2/cmd/integration-test/websocket.go | 2 +- v2/cmd/integration-test/workflow.go | 2 +- v2/internal/runner/runner.go | 22 +++---- v2/internal/runner/update_test.go | 2 +- v2/pkg/core/engine.go | 4 +- v2/pkg/core/execute.go | 8 +-- v2/pkg/core/inputs/hybrid/hmap.go | 3 +- v2/pkg/operators/common/dsl/dsl.go | 7 +-- v2/pkg/output/output.go | 35 ----------- v2/pkg/progress/progress.go | 24 ------- .../protocols/common/generators/generators.go | 10 +-- .../protocols/common/generators/validate.go | 2 +- v2/pkg/protocols/dns/dns_test.go | 2 +- v2/pkg/protocols/dns/operators_test.go | 2 +- v2/pkg/protocols/dns/request.go | 6 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/file_test.go | 2 +- v2/pkg/protocols/file/find_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 2 +- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/headless/request.go | 8 +-- v2/pkg/protocols/http/build_request_test.go | 2 +- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/http/http_test.go | 2 +- v2/pkg/protocols/http/operators_test.go | 2 +- v2/pkg/protocols/http/request.go | 4 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/network/network_test.go | 2 +- v2/pkg/protocols/network/operators_test.go | 2 +- v2/pkg/protocols/network/request.go | 18 +++--- v2/pkg/protocols/network/request_test.go | 2 +- v2/pkg/protocols/offlinehttp/find_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 2 +- v2/pkg/protocols/protocols.go | 3 + v2/pkg/protocols/ssl/ssl.go | 46 ++++++++++---- v2/pkg/protocols/ssl/ssl_test.go | 2 +- v2/pkg/protocols/websocket/websocket.go | 61 +++++++++--------- v2/pkg/templates/templates.go | 4 +- v2/{internal => pkg}/testutils/integration.go | 0 v2/{internal => pkg}/testutils/testutils.go | 62 ++++++++++++++++++- 47 files changed, 203 insertions(+), 183 deletions(-) rename v2/{internal => pkg}/testutils/integration.go (100%) rename v2/{internal => pkg}/testutils/testutils.go (57%) diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index fc96fde80..7f05014c1 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -11,7 +11,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var ( diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 3e1ae8146..5bb2ed2c4 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -1,7 +1,7 @@ package main import ( - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var dnsTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a81111c4c..8805fef81 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -11,7 +11,7 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var httpTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index b854e0a45..47005ed18 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/logrusorgru/aurora" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var ( diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 8c44bc20a..e07c45c21 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "net/http" "net/http/httptest" "os" "strings" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var loaderTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index e170b07b9..ac34c5fd5 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -3,7 +3,7 @@ package main import ( "net" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var networkTestcases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index be20ba773..b5b0e5a34 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/gobwas/ws/wsutil" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var websocketTestCases = map[string]testutils.TestCase{ diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 5f39b4ebf..31202d090 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -7,7 +7,7 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) var workflowTestcases = map[string]testutils.TestCase{ diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index fb57b32b3..dec4d16bd 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -145,18 +145,16 @@ func New(options *types.Options) (*Runner, error) { } if !options.NoInteractsh { - interactshClient, err := interactsh.New(&interactsh.Options{ - ServerURL: options.InteractshURL, - Authorization: options.InteractshToken, - CacheSize: int64(options.InteractionsCacheSize), - Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second, - PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, - Output: runner.output, - IssuesClient: runner.issuesClient, - Progress: runner.progress, - Debug: runner.options.Debug, - }) + opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts.Debug = runner.options.Debug + opts.ServerURL = options.InteractshURL + opts.Authorization = options.InteractshToken + opts.CacheSize = int64(options.InteractionsCacheSize) + opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsColldownPeriod) * time.Second + opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + + interactshClient, err := interactsh.New(opts) if err != nil { gologger.Error().Msgf("Could not create interactsh client: %s", err) } else { diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index dc57140c2..bc0888341 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -13,8 +13,8 @@ import ( "testing" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/stretchr/testify/require" ) diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index 5d7cf9461..4ea9b5417 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -27,8 +27,8 @@ type Engine struct { type InputProvider interface { // Count returns the number of items for input provider Count() int64 - // Scan iterates the input and for each found item calls the callback - // until the input provider has been exhausted. + // Scan iterates the input and each found item is passed to the + // callback consumer. Scan(callback func(value string)) } diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index ee312b0cc..4408d289d 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -17,12 +17,12 @@ import ( // // All the execution logic for the templates/workflows happens in this part // of the engine. -func (e *Engine) Execute(templates []*templates.Template, input InputProvider) *atomic.Bool { - return e.ExecuteWithOpts(templates, input, false) +func (e *Engine) Execute(templates []*templates.Template, target InputProvider) *atomic.Bool { + return e.ExecuteWithOpts(templates, target, false) } // ExecuteWithOpts is execute with the full options -func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input InputProvider, noCluster bool) *atomic.Bool { +func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { finalTemplates, _ = e.ClusterTemplates(templatesList) @@ -48,7 +48,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, input Inpu e.executeSelfContainedTemplateWithInput(template, results) default: // All other request types are executed here - e.executeModelWithInput(templateType, template, input, results) + e.executeModelWithInput(templateType, template, target, results) } wg.Done() } diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 1eb4f5e42..406331fe0 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -119,7 +119,8 @@ func (i *Input) Count() int64 { return i.inputCount } -// Scan calls an input provider till the callback is exhausted +// Scan iterates the input and each found item is passed to the +// callback consumer. func (i *Input) Scan(callback func(value string)) { callbackFunc := func(k, _ []byte) error { callback(string(k)) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 4e7ffa2c1..cfeede8f0 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -332,7 +332,7 @@ var functions = map[string]govaluate.ExpressionFunction{ } now := time.Now() offset := now.Add(time.Duration(seconds) * time.Second) - return offset.Unix(), nil + return float64(offset.Unix()), nil }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { @@ -363,11 +363,6 @@ var functions = map[string]govaluate.ExpressionFunction{ gologger.Info().Msgf("print_debug value: %s", fmt.Sprint(args)) return true, nil }, - // is_before_now compares a timestamp and returns true if the first - // passed argument is a time.Time that has already passed. - "time_now": func(args ...interface{}) (interface{}, error) { - return float64(time.Now().Unix()), nil - }, } // HelperFunctions returns the dsl helper functions diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 96850cb05..991c99232 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -224,38 +224,3 @@ func (w *StandardWriter) Close() { w.errorFile.Close() } } - -// MockOutputWriter is a mocked output writer. -type MockOutputWriter struct { - aurora aurora.Aurora - RequestCallback func(templateID, url, requestType string, err error) - WriteCallback func(o *ResultEvent) -} - -// NewMockOutputWriter creates a new mock output writer -func NewMockOutputWriter() *MockOutputWriter { - return &MockOutputWriter{aurora: aurora.NewAurora(false)} -} - -// Close closes the output writer interface -func (m *MockOutputWriter) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *MockOutputWriter) Colorizer() aurora.Aurora { - return m.aurora -} - -// Write writes the event to file and/or screen. -func (m *MockOutputWriter) Write(result *ResultEvent) error { - if m.WriteCallback != nil { - m.WriteCallback(result) - } - return nil -} - -// Request writes a log the requests trace log -func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { - if m.RequestCallback != nil { - m.RequestCallback(templateID, url, requestType, err) - } -} diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index fcd34340d..220ff49a6 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -247,27 +247,3 @@ func (p *StatsTicker) Stop() { _ = p.server.Shutdown(context.Background()) } } - -type MockProgressClient struct{} - -// Stop stops the progress recorder. -func (m *MockProgressClient) Stop() {} - -// Init inits the progress bar with initial details for scan -func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} - -// AddToTotal adds a value to the total request count -func (m *MockProgressClient) AddToTotal(delta int64) {} - -// IncrementRequests increments the requests counter by 1. -func (m *MockProgressClient) IncrementRequests() {} - -// IncrementMatched increments the matched counter by 1. -func (m *MockProgressClient) IncrementMatched() {} - -// IncrementErrorsBy increments the error counter by count. -func (m *MockProgressClient) IncrementErrorsBy(count int64) {} - -// IncrementFailedRequestsBy increments the number of requests counter by count -// along with errors. -func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 47b2fa44a..f3aaa8a34 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -7,14 +7,14 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) -// Generator is the generator struct for generating payloads -type Generator struct { +// PayloadGenerator is the generator struct for generating payloads +type PayloadGenerator struct { Type AttackType payloads map[string][]string } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*Generator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog *catalog.Catalog) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -35,7 +35,7 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &Generator{} + generator := &PayloadGenerator{} if err := generator.validate(payloads, templatePath); err != nil { return nil, err } @@ -66,7 +66,7 @@ type Iterator struct { } // NewIterator creates a new iterator for the payloads generator -func (g *Generator) NewIterator() *Iterator { +func (g *PayloadGenerator) NewIterator() *Iterator { var payloads []*payloadIterator for name, values := range g.payloads { diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 0384f8907..294ddca3a 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -11,7 +11,7 @@ import ( ) // validate validates the payloads if any. -func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error { +func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePath string) error { for name, payload := range payloads { switch pt := payload.(type) { case string: diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index d01f84bdc..b0f8dddfe 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestGenerateDNSVariables(t *testing.T) { diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 0e4c6ff3e..e5618f610 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -8,13 +8,13 @@ import ( "github.com/miekg/dns" "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 05d4930da..2c9165679 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -35,7 +35,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Compile each request for the template based on the URL compiledRequest, err := request.Make(domain) if err != nil { - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } @@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review // Send the request to the target servers response, err := request.dnsClient.Do(compiledRequest) if err != nil { - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) } if response == nil { @@ -61,7 +61,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } request.options.Progress.IncrementRequests() - request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index bc5f230b3..d2ed83994 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestDNSExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/file/file_test.go b/v2/pkg/protocols/file/file_test.go index 7338e1461..d568f9810 100644 --- a/v2/pkg/protocols/file/file_test.go +++ b/v2/pkg/protocols/file/file_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFileCompile(t *testing.T) { diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go index a2e2e9f36..7262bb26a 100644 --- a/v2/pkg/protocols/file/find_test.go +++ b/v2/pkg/protocols/file/find_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFindInputPaths(t *testing.T) { diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index da40c76d4..f85ff6c82 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index ef650bab8..18e156550 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -74,7 +74,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index 0c9f7e9cc..bccfae728 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -8,13 +8,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFileExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index dc1d3467c..89de2eaa9 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -26,7 +26,7 @@ func (request *Request) Type() templateTypes.ProtocolType { func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { instance, err := request.options.Browser.NewInstance() if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } @@ -34,19 +34,19 @@ func (request *Request) ExecuteWithResults(inputURL string, metadata, previous o parsedURL, err := url.Parse(inputURL) if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } out, page, err := instance.Run(parsedURL, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", err) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - request.options.Output.Request(request.options.TemplatePath, inputURL, "headless", nil) + request.options.Output.Request(request.options.TemplatePath, inputURL, request.Type().String(), nil) request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", inputURL) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index c5cc3aaff..194463932 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -6,10 +6,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index df79e8f47..f85e49c2c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -131,7 +131,7 @@ type Request struct { options *protocols.ExecuterOptions totalRequests int customHeaders map[string]string - generator *generators.Generator // optional, only enabled when using payloads + generator *generators.PayloadGenerator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client dynamicValues map[string]interface{} diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 69a71fb29..4da40ce6f 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -5,10 +5,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestHTTPCompile(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index e2ecbdf57..bc7b888a2 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -7,13 +7,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index af5075f39..3a455472a 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -363,7 +363,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send @@ -401,7 +401,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) - request.options.Output.Request(request.options.TemplatePath, formedURL, "http", err) + request.options.Output.Request(request.options.TemplatePath, formedURL, request.Type().String(), err) duration := time.Since(timeStart) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 6b3cbe2a6..83c9ac813 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -75,7 +75,7 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` - generator *generators.Generator + generator *generators.PayloadGenerator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer options *protocols.ExecuterOptions diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 180b34192..26b83502d 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -5,9 +5,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestNetworkCompileMake(t *testing.T) { diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 735d9af38..9e8a193c3 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -5,13 +5,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ed52117fb..14b563f32 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -42,7 +42,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review address, err = getAddress(input) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, "network", err) + request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -71,7 +71,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return err } @@ -119,7 +119,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } @@ -149,7 +149,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input data = []byte(input.Data) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -157,7 +157,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", dataErr) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -168,7 +168,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input return nil } if _, err := conn.Write(finalData); err != nil { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -203,7 +203,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } } - request.options.Output.Request(request.options.TemplatePath, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplatePath, actualAddress, request.Type().String(), err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 @@ -234,7 +234,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) if err != nil && !os.IsTimeout(err) { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) closeTimer(readInterval) return errors.Wrap(err, "could not read from server") } @@ -247,7 +247,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input final = make([]byte, bufferSize) n, err = conn.Read(final) if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplatePath, address, "network", err) + request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index d1f266b48..8b44091e2 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -10,13 +10,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestNetworkExecuteWithResults(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/find_test.go b/v2/pkg/protocols/offlinehttp/find_test.go index 369ef4405..d6b933ce2 100644 --- a/v2/pkg/protocols/offlinehttp/find_test.go +++ b/v2/pkg/protocols/offlinehttp/find_test.go @@ -8,10 +8,10 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestFindResponses(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 2f4b91fd5..8669d8b58 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -7,13 +7,13 @@ import ( "github.com/stretchr/testify/require" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) func TestResponseToDSLMap(t *testing.T) { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index a686b9c45..51cfab061 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -96,6 +97,8 @@ type Request interface { MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent // GetCompiledOperators returns a list of the compiled operators GetCompiledOperators() []*operators.Operators + // Type returns the type of the protocol request + Type() templateTypes.ProtocolType } // OutputEventCallback is a callback event for any results found during scanning. diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 8f624d5b7..ba26c8e30 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" @@ -30,6 +31,9 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -72,28 +76,45 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous if err != nil { return nil } - hostname, _, _ := net.SplitHostPort(address) + hostname, port, _ := net.SplitHostPort(address) + requestOptions := request.options + payloadValues := make(map[string]interface{}) + for k, v := range dynamicValues { + payloadValues[k] = v + } + payloadValues["Hostname"] = address + payloadValues["Host"] = hostname + payloadValues["Port"] = port + + finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) + if dataErr != nil { + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) + return errors.Wrap(dataErr, "could not evaluate template expressions") + } + + addressToDial := string(finalAddress) config := &tls.Config{InsecureSkipVerify: true, ServerName: hostname} - conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", address, config) + conn, err := request.dialer.DialTLSWithConfig(context.Background(), "tcp", addressToDial, config) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "ssl", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(requestOptions.Options.Timeout) * time.Second)) connTLS, ok := conn.(*tls.Conn) if !ok { return nil } - request.options.Output.Request(request.options.TemplateID, address, "ssl", err) + requestOptions.Output.Request(requestOptions.TemplateID, address, request.Type().String(), err) gologger.Verbose().Msgf("Sent SSL request to %s", address) - if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", request.options.TemplateID, input) + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { + gologger.Debug().Str("address", input).Msgf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input) } state := connTLS.ConnectionState() @@ -110,13 +131,14 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous data["response"] = jsonDataString data["host"] = input + data["matched"] = addressToDial data["not_after"] = float64(cert.NotAfter.Unix()) data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) - if request.options.Options.Debug || request.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped SSL response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) + event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) + if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) } callback(event) return nil diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index fef44fbbc..10522d6a8 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -3,10 +3,10 @@ package ssl import ( "testing" - "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/stretchr/testify/require" ) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 7731dc29c..7d2aca246 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -65,7 +65,7 @@ type Request struct { // be provided as payload which will be read on run-time. Payloads map[string]interface{} `yaml:"payloads,omitempty" jsonschema:"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request"` - generator *generators.Generator + generator *generators.PayloadGenerator // cache any variables that may be needed for operation. dialer *fastdialer.Dialer @@ -174,46 +174,44 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam payloadValues["Scheme"] = parsed.Scheme payloadValues["Path"] = parsed.Path + requestOptions := request.options for key, value := range request.Headers { finalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } header.Set(key, string(finalData)) } websocketDialer := ws.Dialer{ Header: ws.HandshakeHeaderHTTP(header), - Timeout: time.Duration(request.options.Options.Timeout) * time.Second, + Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second, NetDial: request.dialer.Dial, TLSConfig: &tls.Config{InsecureSkipVerify: true, ServerName: hostname}, } finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } addressToDial := string(finalAddress) parsedAddress, err := url.Parse(addressToDial) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not parse input url") } - - if parsed.Path != "" && parsed.Path != "/" { - parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) - addressToDial = parsedAddress.String() - } + parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) + addressToDial = parsedAddress.String() conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } defer conn.Close() @@ -225,18 +223,18 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam events, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not read write response") } - request.options.Progress.IncrementRequests() + requestOptions.Progress.IncrementRequests() - if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", request.options.TemplateID, input) + if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { + gologger.Debug().Str("address", input).Msgf("[%s] Dumped Websocket request for %s", requestOptions.TemplateID, input) gologger.Print().Msgf("%s", requestOutput) } - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) gologger.Verbose().Msgf("Sent Websocket request to %s", input) data := make(map[string]interface{}) @@ -253,13 +251,13 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["matched"] = addressToDial data["ip"] = request.dialer.GetDialedIP(hostname) - event := eventcreator.CreateEventWithAdditionalOptions(request, data, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues }) - if request.options.Options.Debug || request.options.Options.DebugResponse { + if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", request.options.TemplateID, input) - gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, request.options.Options.NoColor, false)) + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", requestOptions.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, requestOptions.Options.NoColor, false)) } callback(event) @@ -270,28 +268,29 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map reqBuilder := &strings.Builder{} inputEvents := make(map[string]interface{}) + requestOptions := request.options for _, req := range request.Inputs { reqBuilder.Grow(len(req.Data)) finalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues) if dataErr != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", dataErr) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(dataErr, "could not evaluate template expressions") } reqBuilder.WriteString(string(finalData)) err = wsutil.WriteClientMessage(conn, ws.OpText, finalData) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } msg, opCode, err := wsutil.ReadServerData(conn) if err != nil { - request.options.Output.Request(request.options.TemplateID, input, "websocket", err) - request.options.Progress.IncrementFailedRequestsBy(1) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } // Only perform matching and writes in case we recieve diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d44ac8e7d..39e0e447d 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -83,8 +83,8 @@ type Template struct { Path string `yaml:"-" json:"-"` } -// TemplateTypes is a list of accepted template types -var TemplateTypes = []string{ +// TemplateProtocols is a list of accepted template protocols +var TemplateProtocols = []string{ "dns", "file", "http", diff --git a/v2/internal/testutils/integration.go b/v2/pkg/testutils/integration.go similarity index 100% rename from v2/internal/testutils/integration.go rename to v2/pkg/testutils/integration.go diff --git a/v2/internal/testutils/testutils.go b/v2/pkg/testutils/testutils.go similarity index 57% rename from v2/internal/testutils/testutils.go rename to v2/pkg/testutils/testutils.go index e58c70791..0ea388f7a 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -3,6 +3,7 @@ package testutils import ( "go.uber.org/ratelimit" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -73,7 +74,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco TemplateID: info.ID, TemplateInfo: info.Info, TemplatePath: info.Path, - Output: output.NewMockOutputWriter(), + Output: NewMockOutputWriter(), Options: options, Progress: progressImpl, ProjectFile: nil, @@ -90,3 +91,62 @@ type NoopWriter struct{} // Write writes the data to an output writer. func (n *NoopWriter) Write(data []byte, level levels.Level) {} + +// MockOutputWriter is a mocked output writer. +type MockOutputWriter struct { + aurora aurora.Aurora + RequestCallback func(templateID, url, requestType string, err error) + WriteCallback func(o *output.ResultEvent) +} + +// NewMockOutputWriter creates a new mock output writer +func NewMockOutputWriter() *MockOutputWriter { + return &MockOutputWriter{aurora: aurora.NewAurora(false)} +} + +// Close closes the output writer interface +func (m *MockOutputWriter) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *MockOutputWriter) Colorizer() aurora.Aurora { + return m.aurora +} + +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) Write(result *output.ResultEvent) error { + if m.WriteCallback != nil { + m.WriteCallback(result) + } + return nil +} + +// Request writes a log the requests trace log +func (m *MockOutputWriter) Request(templateID, url, requestType string, err error) { + if m.RequestCallback != nil { + m.RequestCallback(templateID, url, requestType, err) + } +} + +type MockProgressClient struct{} + +// Stop stops the progress recorder. +func (m *MockProgressClient) Stop() {} + +// Init inits the progress bar with initial details for scan +func (m *MockProgressClient) Init(hostCount int64, rulesCount int, requestCount int64) {} + +// AddToTotal adds a value to the total request count +func (m *MockProgressClient) AddToTotal(delta int64) {} + +// IncrementRequests increments the requests counter by 1. +func (m *MockProgressClient) IncrementRequests() {} + +// IncrementMatched increments the matched counter by 1. +func (m *MockProgressClient) IncrementMatched() {} + +// IncrementErrorsBy increments the error counter by count. +func (m *MockProgressClient) IncrementErrorsBy(count int64) {} + +// IncrementFailedRequestsBy increments the number of requests counter by count +// along with errors. +func (m *MockProgressClient) IncrementFailedRequestsBy(count int64) {} From 4d8eaad0a3050a308f308e168b85b6537e5bab56 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Fri, 5 Nov 2021 15:27:49 +0530 Subject: [PATCH 090/196] add unit test for unique interactsh url #1068 --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 2 +- v2/internal/testutils/testutils.go | 77 ++++++++++--------- .../protocols/common/interactsh/interactsh.go | 6 +- v2/pkg/protocols/http/build_request_test.go | 44 ++++++++++- v2/pkg/types/types.go | 4 +- 6 files changed, 88 insertions(+), 47 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 7f43868f4..16997964d 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -106,7 +106,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), - flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index de1caff93..c9a0ab846 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -249,7 +249,7 @@ func New(options *types.Options) (*Runner, error) { Authorization: options.InteractshToken, CacheSize: int64(options.InteractionsCacheSize), Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsColldownPeriod) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, Output: runner.output, IssuesClient: runner.issuesClient, diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630..79e426422 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -22,42 +22,47 @@ func Init(options *types.Options) { // DefaultOptions is the default options structure for nuclei during mocking. var DefaultOptions = &types.Options{ - Metrics: false, - Debug: false, - DebugRequests: false, - DebugResponse: false, - Silent: false, - Version: false, - Verbose: false, - NoColor: true, - UpdateTemplates: false, - JSON: false, - JSONRequests: false, - EnableProgressBar: false, - TemplatesVersion: false, - TemplateList: false, - Stdin: false, - StopAtFirstMatch: false, - NoMeta: false, - Project: false, - MetricsPort: 0, - BulkSize: 25, - TemplateThreads: 10, - Timeout: 5, - Retries: 1, - RateLimit: 150, - ProjectPath: "", - Severities: severity.Severities{}, - Targets: []string{}, - TargetsFilePath: "", - Output: "", - ProxyURL: "", - ProxySocksURL: "", - TemplatesDirectory: "", - TraceLogFile: "", - Templates: []string{}, - ExcludedTemplates: []string{}, - CustomHeaders: []string{}, + Metrics: false, + Debug: false, + DebugRequests: false, + DebugResponse: false, + Silent: false, + Version: false, + Verbose: false, + NoColor: true, + UpdateTemplates: false, + JSON: false, + JSONRequests: false, + EnableProgressBar: false, + TemplatesVersion: false, + TemplateList: false, + Stdin: false, + StopAtFirstMatch: false, + NoMeta: false, + Project: false, + MetricsPort: 0, + BulkSize: 25, + TemplateThreads: 10, + Timeout: 5, + Retries: 1, + RateLimit: 150, + ProjectPath: "", + Severities: severity.Severities{}, + Targets: []string{}, + TargetsFilePath: "", + Output: "", + ProxyURL: "", + ProxySocksURL: "", + TemplatesDirectory: "", + TraceLogFile: "", + Templates: []string{}, + ExcludedTemplates: []string{}, + CustomHeaders: []string{}, + InteractshURL: "https://interactsh.com", + InteractionsCacheSize: 5000, + InteractionsEviction: 60, + InteractionsCooldownPeriod: 5, + InteractionsPollDuration: 5, } // MockOutputWriter is a mocked output writer. diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 5fff7ac50..2fc788577 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -241,16 +241,12 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { c.requests.Set(id, data, c.eviction) return } - matched := false for _, interaction := range interactions { if c.processInteractionForRequest(interaction, data) { - matched = true + c.interactions.Delete(id) break } } - if matched { - c.interactions.Delete(id) - } } else { c.requests.Set(id, data, c.eviction) } diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 122d81bce..663fe1097 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -3,12 +3,13 @@ package http import ( "net/url" "testing" - - "github.com/stretchr/testify/require" + "time" "github.com/projectdiscovery/nuclei/v2/internal/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/stretchr/testify/require" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { @@ -198,3 +199,42 @@ Accept-Encoding: gzip`}, authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw") } + +func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { + + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-unique-interactsh" + request := &Request{ + ID: templateID, + Name: "testing", + Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, + Method: "GET", + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile http request") + + generator := request.newGenerator() + + generator.options.Interactsh, err = interactsh.New(&interactsh.Options{ + ServerURL: options.InteractshURL, + CacheSize: int64(options.InteractionsCacheSize), + Eviction: time.Duration(options.InteractionsEviction) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, + PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, + }) + require.Nil(t, err, "could not create interactsh client") + + got, err := generator.Make("https://example.com", map[string]interface{}{}) + require.Nil(t, err, "could not make http request") + + //check if all the interactsh markers are replaced with unique urls + require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url") + //check the length of returned urls + require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url") +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c19ad18a6..50a8c78a0 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -100,9 +100,9 @@ type Options struct { // Eviction is the number of seconds after which to automatically discard // interaction requests. InteractionsEviction int - // InteractionsColldownPeriod is additional seconds to wait for interactions after closing + // InteractionsCooldownPeriod is additional seconds to wait for interactions after closing // of the poller. - InteractionsColldownPeriod int + InteractionsCooldownPeriod int // OfflineHTTP is a flag that specific offline processing of http response // using same matchers/extractors from http protocol without the need // to send a new request, reading responses from a file. From 0a09b9e451ddbdb4e047ed575aa563ececdb92b6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 16:59:24 +0530 Subject: [PATCH 091/196] Added test + misc --- v2/pkg/protocols/ssl/ssl_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 10522d6a8..8d7e02179 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -15,7 +15,9 @@ func TestSSLProtocol(t *testing.T) { testutils.Init(options) templateID := "testing-ssl" - request := &Request{} + request := &Request{ + Address: "{{Hostname}}", + } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, @@ -23,8 +25,12 @@ func TestSSLProtocol(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile ssl request") - err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) {}) + var gotEvent output.InternalEvent + err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) { + gotEvent = event.InternalEvent + }) require.Nil(t, err, "could not run ssl request") + require.NotEmpty(t, gotEvent, "could not get event items") } func TestGetAddress(t *testing.T) { From 1c360e0344e1ca2c7106f4519602f1e59925b42d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 17:04:42 +0530 Subject: [PATCH 092/196] Fixed DSL test --- v2/pkg/operators/common/dsl/dsl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 42bf5810d..3db529770 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -21,7 +21,7 @@ func TestDSLURLEncodeDecode(t *testing.T) { } func TestDSLTimeComparison(t *testing.T) { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions("time_now() > not_after", HelperFunctions()) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("unixtime() > not_after", HelperFunctions()) require.Nil(t, err, "could not compare time") result, err := compiled.Evaluate(map[string]interface{}{"not_after": float64(time.Now().Unix() - 1000)}) From e8197f127f1a29ba46287a59fa660628e2e0d3b9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 5 Nov 2021 17:24:23 +0530 Subject: [PATCH 093/196] Fixed rmeote url loader test cases --- v2/pkg/catalog/loader/loader.go | 7 +++++-- v2/pkg/catalog/loader/remote_loader.go | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index a2d7d65fc..1a61cb25b 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -56,6 +56,8 @@ func NewConfig(options *types.Options, catalog *catalog.Catalog, executerOpts pr loaderConfig := Config{ Templates: options.Templates, Workflows: options.Workflows, + TemplateURLs: options.TemplateURLs, + WorkflowURLs: options.WorkflowURLs, ExcludeTemplates: options.ExcludedTemplates, Tags: options.Tags, ExcludeTags: options.ExcludeTags, @@ -96,7 +98,8 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } - if len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 { + urlbasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 + if urlbasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) if err != nil { return store, err @@ -106,7 +109,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory - if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 { + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlbasedTemplatesProvided { store.finalTemplates = []string{config.TemplatesDirectory} } diff --git a/v2/pkg/catalog/loader/remote_loader.go b/v2/pkg/catalog/loader/remote_loader.go index d8f71eda1..c787e9601 100644 --- a/v2/pkg/catalog/loader/remote_loader.go +++ b/v2/pkg/catalog/loader/remote_loader.go @@ -3,9 +3,10 @@ package loader import ( "bufio" "fmt" - "github.com/pkg/errors" "net/http" "strings" + + "github.com/pkg/errors" ) type ContentType string From 6d0d3e1c359457bb22ee3afe380c8ebd96b9d4ee Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Fri, 5 Nov 2021 20:00:46 +0530 Subject: [PATCH 094/196] add unique check to test case --- v2/pkg/protocols/http/build_request_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 663fe1097..0109498a6 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -233,8 +233,22 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { got, err := generator.Make("https://example.com", map[string]interface{}{}) require.Nil(t, err, "could not make http request") - //check if all the interactsh markers are replaced with unique urls + // check if all the interactsh markers are replaced with unique urls require.NotContains(t, got.request.URL.String(), "{{interactsh-url}}", "could not get correct interactsh url") - //check the length of returned urls + // check the length of returned urls require.Equal(t, len(got.interactshURLs), 4, "could not get correct interactsh url") + // check if the interactsh urls are unique + require.True(t, areUnique(got.interactshURLs), "interactsh urls are not unique") +} + +// areUnique checks if the elements of string slice are unique +func areUnique(elements []string) bool { + encountered := map[string]bool{} + for v := range elements { + if encountered[elements[v]] { + return false + } + encountered[elements[v]] = true + } + return true } From af68a30c5c355a58675ab99ca8686250f77595e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 05:04:43 +0000 Subject: [PATCH 095/196] chore(deps): bump golang from 1.17.2-alpine to 1.17.3-alpine Bumps golang from 1.17.2-alpine to 1.17.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7d3467d34..c5c46e839 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-alpine as build-env +FROM golang:1.17.3-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.14 From d772dedef146bf178be399fd163f6006206260d7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:01:45 +0530 Subject: [PATCH 096/196] Make default part for new protocols to response --- v2/pkg/protocols/protocols.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 51cfab061..8333ec865 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -134,7 +134,12 @@ func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEven // MakeDefaultExtractFunc performs extracting operation for an extractor on model and returns true or false. func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - item, ok := data[extractor.Part] + part := extractor.Part + if part == "" { + part = "response" + } + + item, ok := data[part] if !ok { return nil } @@ -155,6 +160,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E // MakeDefaultMatchFunc performs matching operation for a matcher on model and returns true or false. func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + part := matcher.Part + if part == "" { + part = "response" + } + partItem, ok := data[matcher.Part] if !ok && len(matcher.DSL) == 0 { return false, nil From ce7534112aa392765a68659642c4ef0ec8c6101d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:10:04 +0530 Subject: [PATCH 097/196] Fixed linter issues --- v2/pkg/protocols/protocols.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 8333ec865..7ad9d24aa 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -165,7 +165,7 @@ func MakeDefaultMatchFunc(data map[string]interface{}, matcher *matchers.Matcher part = "response" } - partItem, ok := data[matcher.Part] + partItem, ok := data[part] if !ok && len(matcher.DSL) == 0 { return false, nil } From d442c51c1a15e2302773a5d9f740a91eb280b308 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:14:47 +0530 Subject: [PATCH 098/196] Changed input to target in core --- v2/pkg/core/execute.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 4408d289d..c9d36cd97 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -66,10 +66,10 @@ func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Templ } // executeModelWithInput executes a type of template with input -func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template *templates.Template, input InputProvider, results *atomic.Bool) { +func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool) { wg := e.workPool.InputPool(templateType) - input.Scan(func(scannedValue string) { + target.Scan(func(scannedValue string) { // Skip if the host has had errors if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { return From 88a296c9a7887f4e105d204031221f2dd6d48d4e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 16:24:47 +0530 Subject: [PATCH 099/196] Adding gzip encoding helpers --- v2/pkg/operators/common/dsl/dsl.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index cfeede8f0..378b171c6 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -1,6 +1,8 @@ package dsl import ( + "bytes" + "compress/gzip" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -120,6 +122,16 @@ var functions = map[string]govaluate.ExpressionFunction{ return sEnc, nil }, + "gzip": func(args ...interface{}) (interface{}, error) { + if len(args) != 1 { + return nil, ErrDSLArguments + } + buffer := &bytes.Buffer{} + if _, err := gzip.NewWriter(buffer).Write([]byte(args[0].(string))); err != nil { + return "", err + } + return buffer.String(), nil + }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" "base64_py": func(args ...interface{}) (interface{}, error) { if len(args) != 1 { From 7d0529aaf7d972faf9ba1e10748ce9d0f6ca4393 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 17:39:08 +0530 Subject: [PATCH 100/196] Fixed gzip encoding bug + added test case --- v2/pkg/operators/common/dsl/dsl.go | 5 ++++- v2/pkg/operators/common/dsl/dsl_test.go | 17 +++++++++++++++++ .../common/helpers/deserialization/java.go | 8 ++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 378b171c6..e4cee9c5e 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -127,9 +127,12 @@ var functions = map[string]govaluate.ExpressionFunction{ return nil, ErrDSLArguments } buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write([]byte(args[0].(string))); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write([]byte(args[0].(string))); err != nil { return "", err } + _ = writer.Close() + return buffer.String(), nil }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index 3db529770..f75baf087 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -1,10 +1,14 @@ package dsl import ( + "compress/gzip" + "io/ioutil" + "strings" "testing" "time" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/stretchr/testify/require" ) @@ -28,3 +32,16 @@ func TestDSLTimeComparison(t *testing.T) { require.Nil(t, err, "could not evaluate compare time") require.Equal(t, true, result, "could not get url encoded data") } + +func TestDSLGzipSerialize(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("gzip(\"hello world\")", HelperFunctions()) + require.Nil(t, err, "could not compare time") + + result, err := compiled.Evaluate(make(map[string]interface{})) + require.Nil(t, err, "could not evaluate compare time") + + reader, _ := gzip.NewReader(strings.NewReader(types.ToString(result))) + data, _ := ioutil.ReadAll(reader) + + require.Equal(t, "hello world", string(data), "could not get gzip encoded data") +} diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go index ee8ddfb6b..0ea107c37 100644 --- a/v2/pkg/protocols/common/helpers/deserialization/java.go +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -48,15 +48,19 @@ func gadgetEncodingHelper(returnData []byte, encoding string) string { return hex.EncodeToString(returnData) case "gzip": buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write(returnData); err != nil { return "" } + _ = writer.Close() return buffer.String() case "gzip-base64": buffer := &bytes.Buffer{} - if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + writer := gzip.NewWriter(buffer) + if _, err := writer.Write(returnData); err != nil { return "" } + _ = writer.Close() return urlsafeBase64Encode(buffer.Bytes()) case "base64-raw": return base64.StdEncoding.EncodeToString(returnData) From 5e6b6c6b14c5542f8f8c9b2e13e62d5a9aa3ec53 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Nov 2021 17:56:14 +0530 Subject: [PATCH 101/196] Show errors on invalid URL typo --- v2/pkg/protocols/websocket/websocket.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 7d2aca246..cdc40cc0c 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -201,9 +201,9 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam addressToDial := string(finalAddress) parsedAddress, err := url.Parse(addressToDial) if err != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) - return errors.Wrap(dataErr, "could not parse input url") + return errors.Wrap(err, "could not parse input url") } parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path) addressToDial = parsedAddress.String() From 9675efa680926500fc2be1c84a65805baf6012d0 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 8 Nov 2021 19:33:54 +0100 Subject: [PATCH 102/196] Adding support for custom headers via CLI in raw http requests --- v2/go.mod | 7 ------- v2/go.sum | 4 ---- v2/pkg/protocols/http/build_request.go | 3 +++ v2/pkg/protocols/http/raw/raw.go | 27 ++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 5e6e5f0c7..2c003738b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( - github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c @@ -20,7 +19,6 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.12 - github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible @@ -49,7 +47,6 @@ require ( github.com/shirou/gopsutil/v3 v3.21.7 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 - github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 @@ -71,16 +68,13 @@ require ( require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect - github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect @@ -105,7 +99,6 @@ require ( github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect diff --git a/v2/go.sum b/v2/go.sum index a497178f8..177689ca2 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -42,7 +42,6 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -54,7 +53,6 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -75,7 +73,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= @@ -409,7 +406,6 @@ github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 20f3d4801..4b4ac9f21 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -227,6 +227,9 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest // Unsafe option uses rawhttp library if r.request.Unsafe { + if len(r.options.Options.CustomHeaders) > 0 { + _ = rawRequestData.TryFillCustomHeaders(r.options.Options.CustomHeaders) + } unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request} return unsafeReq, nil } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 18bebc39f..e2d7f8ebe 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -3,6 +3,7 @@ package raw import ( "bufio" "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -146,3 +147,29 @@ func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1) return fixed } + +// TryFillCustomHeaders after the Host header +func (r *Request) TryFillCustomHeaders(headers []string) error { + unsafeBytes := bytes.ToLower(r.UnsafeRawBytes) + // locate first host header + hostHeaderIndex := bytes.Index(unsafeBytes, []byte("host:")) + if hostHeaderIndex > 0 { + // attempt to locate next newline + newLineIndex := bytes.Index(unsafeBytes[hostHeaderIndex:], []byte("\r\n")) + if newLineIndex > 0 { + newLineIndex += hostHeaderIndex + 2 + // insert custom headers + var buf bytes.Buffer + buf.Write(r.UnsafeRawBytes[:newLineIndex]) + for _, header := range headers { + buf.WriteString(fmt.Sprintf("%s\r\n", header)) + } + buf.Write(r.UnsafeRawBytes[newLineIndex:]) + r.UnsafeRawBytes = buf.Bytes() + return nil + } + return errors.New("no new line found at the end of host header") + } + + return errors.New("no host header found") +} From 10e1b09bf32bd3e33d9b8c882bc5bea2e7b4adff Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 8 Nov 2021 19:45:01 +0100 Subject: [PATCH 103/196] tidying go modules --- v2/go.mod | 10 +++++++++- v2/go.sum | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index 2c003738b..39a54ff3d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( + github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c @@ -19,6 +20,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.12 + github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible @@ -47,10 +49,10 @@ require ( github.com/shirou/gopsutil/v3 v3.21.7 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 + github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 - github.com/weppos/publicsuffix-go v0.15.0 github.com/xanzy/go-gitlab v0.50.3 github.com/ysmood/gson v0.6.4 // indirect github.com/ysmood/leakless v0.7.0 // indirect @@ -66,15 +68,20 @@ require ( moul.io/http2curl v1.0.0 ) +require github.com/weppos/publicsuffix-go v0.15.0 + require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect + github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/eggsampler/acme/v3 v3.2.1 // indirect @@ -99,6 +106,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect diff --git a/v2/go.sum b/v2/go.sum index 177689ca2..a497178f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -42,6 +42,7 @@ github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EF github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= @@ -53,6 +54,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -73,6 +75,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-jira v1.14.0 h1:7GT/3qhar2dGJ0kq8w0d63liNyHOnxZsUZ9Pe4+AKBI= github.com/andygrunwald/go-jira v1.14.0/go.mod h1:KMo2f4DgMZA1C9FdImuLc04x4WQhn5derQpnsuBFgqE= @@ -406,6 +409,7 @@ github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= From 155b8cfed9b66eee291986886ad95fa3c66d9ba4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:00:30 +0530 Subject: [PATCH 104/196] fix #1173: perform matching on all redirect responses instead of final --- v2/pkg/protocols/http/request.go | 121 +++++++++++-------------------- v2/pkg/protocols/http/utils.go | 94 ++++++++++++++---------- 2 files changed, 100 insertions(+), 115 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3f94c81f2..16be32a61 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -400,7 +400,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return errors.Wrap(err, "could not dump http response") } - var data, redirectedResponse []byte + var dumpedResponse []redirectedResponse // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader @@ -409,7 +409,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } else { bodyReader = resp.Body } - data, err = ioutil.ReadAll(bodyReader) + data, err := ioutil.ReadAll(bodyReader) if err != nil { // Ignore body read due to server misconfiguration errors if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { @@ -420,96 +420,61 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } resp.Body.Close() - redirectedResponse, err = dumpResponseWithRedirectChain(resp, data) + dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) if err != nil { return errors.Wrap(err, "could not read http response with redirect chain") } } else { - redirectedResponse = dumpedResponseHeaders + dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} } - // net/http doesn't automatically decompress the response body if an - // encoding has been specified by the user in the request so in case we have to - // manually do it. - dataOrig := data - data, err = handleDecompression(resp, data) - // in case of error use original data - if err != nil { - data = dataOrig - } - - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - dumpedResponseBuilder := &bytes.Buffer{} - dumpedResponseBuilder.Write(dumpedResponseHeaders) - dumpedResponseBuilder.Write(data) - dumpedResponse := dumpedResponseBuilder.Bytes() - redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) - - // Decode gbk response content-types - // gb18030 supersedes gb2312 - responseContentType := resp.Header.Get("Content-Type") - if isContentTypeGbk(responseContentType) { - dumpedResponse, err = decodegbk(dumpedResponse) - if err != nil { - return errors.Wrap(err, "could not gbk decode") - } - redirectedResponse, err = decodegbk(redirectedResponse) - if err != nil { - return errors.Wrap(err, "could not gbk decode") + for _, response := range dumpedResponse { + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil { + return errors.Wrap(err, "could not store in project file") + } } - // the uncompressed body needs to be decoded to standard utf8 - data, err = decodegbk(data) - if err != nil { - return errors.Wrap(err, "could not gbk decode") + matchedURL := reqURL + if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { + matchedURL = generatedRequest.rawRequest.FullURL } - } - - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { - return errors.Wrap(err, "could not store in project file") + if generatedRequest.request != nil { + matchedURL = generatedRequest.request.URL.String() } - } + finalEvent := make(output.InternalEvent) - matchedURL := reqURL - if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { - matchedURL = generatedRequest.rawRequest.FullURL - } - if generatedRequest.request != nil { - matchedURL = generatedRequest.request.URL.String() - } - finalEvent := make(output.InternalEvent) - - outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta) - if i := strings.LastIndex(hostname, ":"); i != -1 { - hostname = hostname[:i] - } - outputEvent["curl-command"] = curlCommand - outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) - outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) - for k, v := range previousEvent { - finalEvent[k] = v - } - for k, v := range outputEvent { - finalEvent[k] = v - } - // Add to history the current request number metadata if asked by the user. - if request.ReqCondition { + outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + if i := strings.LastIndex(hostname, ":"); i != -1 { + hostname = hostname[:i] + } + outputEvent["curl-command"] = curlCommand + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + for k, v := range previousEvent { + finalEvent[k] = v + } for k, v := range outputEvent { - key := fmt.Sprintf("%s_%d", k, requestCount) - previousEvent[key] = v - finalEvent[key] = v + finalEvent[k] = v } + // Add to history the current request number metadata if asked by the user. + if request.ReqCondition { + for k, v := range outputEvent { + key := fmt.Sprintf("%s_%d", k, requestCount) + previousEvent[key] = v + finalEvent[key] = v + } + } + + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta + }) + + responseContentType := resp.Header.Get("Content-Type") + dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType) + + callback(event) } - - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { - internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta - }) - - dumpResponse(event, request.options, redirectedResponse, formedURL, responseContentType) - - callback(event) return nil } diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 89c35a233..1bd25d385 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -10,14 +10,20 @@ import ( "net/http/httputil" "strings" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) +type redirectedResponse struct { + headers []byte + body []byte + fullResponse []byte +} + // dumpResponseWithRedirectChain dumps a http response with the // complete http redirect chain. // @@ -25,18 +31,22 @@ import ( // and returns the data to the user for matching and viewing in that order. // // Inspired from - https://github.com/ffuf/ffuf/issues/324#issuecomment-719858923 -func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, error) { - redirects := []string{} +func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirectedResponse, error) { + var response []redirectedResponse + respData, err := httputil.DumpResponse(resp, false) if err != nil { return nil, err } - redirectChain := &bytes.Buffer{} - - redirectChain.WriteString(tostring.UnsafeToString(respData)) - redirectChain.Write(body) - redirects = append(redirects, redirectChain.String()) - redirectChain.Reset() + respObj := redirectedResponse{ + headers: respData, + body: body, + fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), + } + if err := normalizeResponseBody(resp, respObj); err != nil { + return nil, err + } + response = append(response, respObj) var redirectResp *http.Response if resp != nil && resp.Request != nil { @@ -52,40 +62,50 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]byte, er if redirectResp.Body != nil { body, _ = ioutil.ReadAll(redirectResp.Body) } - redirectChain.WriteString(tostring.UnsafeToString(respData)) - if len(body) > 0 { - redirectChain.WriteString(tostring.UnsafeToString(body)) + respObj := redirectedResponse{ + headers: respData, + body: body, + fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - redirects = append(redirects, redirectChain.String()) + if err := normalizeResponseBody(redirectResp, respObj); err != nil { + return nil, err + } + response = append(response, respObj) redirectResp = redirectResp.Request.Response - redirectChain.Reset() } - for i := len(redirects) - 1; i >= 0; i-- { - redirectChain.WriteString(redirects[i]) - } - return redirectChain.Bytes(), nil + return response, nil } -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - - if i != len(values)-1 { - builder.WriteRune('\n') - builder.WriteString(header) - builder.WriteString(": ") - } - } - builder.WriteRune('\n') +// normalizeResponseBody performs normalization on the http response object. +func normalizeResponseBody(resp *http.Response, response redirectedResponse) error { + var err error + // net/http doesn't automatically decompress the response body if an + // encoding has been specified by the user in the request so in case we have to + // manually do it. + dataOrig := response.body + response.body, err = handleDecompression(resp, response.body) + // in case of error use original data + if err != nil { + response.body = dataOrig } - return builder.String() + response.fullResponse = bytes.ReplaceAll(response.fullResponse, dataOrig, response.body) + + // Decode gbk response content-types + // gb18030 supersedes gb2312 + responseContentType := resp.Header.Get("Content-Type") + if isContentTypeGbk(responseContentType) { + response.fullResponse, err = decodegbk(response.fullResponse) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } + + // the uncompressed body needs to be decoded to standard utf8 + response.body, err = decodegbk(response.body) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } + } + return nil } // dump creates a dump of the http request in form of a byte slice From 8396413f9d7b0c92f3df3e43308ec38ba460185e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:10:07 +0530 Subject: [PATCH 105/196] Added integration test for the redirect chain http functionality --- .../http/get-redirects-chain-headers.yaml | 18 ++++++++++++ v2/cmd/integration-test/http.go | 29 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 integration_tests/http/get-redirects-chain-headers.yaml diff --git a/integration_tests/http/get-redirects-chain-headers.yaml b/integration_tests/http/get-redirects-chain-headers.yaml new file mode 100644 index 000000000..c4446fecd --- /dev/null +++ b/integration_tests/http/get-redirects-chain-headers.yaml @@ -0,0 +1,18 @@ +id: basic-get-redirects-chain-headers + +info: + name: Basic GET Redirects Request With Chain header + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + redirects: true + max-redirects: 3 + matchers: + - type: word + part: header + words: + - "TestRedirectHeaderMatch" \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index a81111c4c..f0d335dd7 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -35,6 +35,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/self-contained.yaml": &httpRequestSelContained{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, + "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, } type httpInteractshRequest struct{} @@ -599,3 +600,31 @@ func (h *httpGetCaseInsensitiveCluster) Execute(filesPath string) error { } return nil } + +type httpGetRedirectsChainHeaders struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + http.Redirect(w, r, "/redirected", http.StatusFound) + }) + router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Header().Set("Secret", "TestRedirectHeaderMatch") + http.Redirect(w, r, "/final", http.StatusFound) + }) + router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.Write([]byte("ok")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} From 3e2875267c419475fea48fd4c4debf4dba7894e1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:12:36 +0530 Subject: [PATCH 106/196] linter fixes --- v2/cmd/integration-test/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index f0d335dd7..bd8f4ac29 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -614,7 +614,7 @@ func (h *httpGetRedirectsChainHeaders) Execute(filePath string) error { http.Redirect(w, r, "/final", http.StatusFound) }) router.GET("/final", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - w.Write([]byte("ok")) + _, _ = w.Write([]byte("ok")) }) ts := httptest.NewServer(router) defer ts.Close() From a9f586369a263c47628a26591e6ed8e008d55e66 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 06:20:10 +0530 Subject: [PATCH 107/196] Misc fixes to request normalization --- v2/pkg/protocols/http/utils.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 1bd25d385..14088eed9 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -43,7 +43,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect body: body, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - if err := normalizeResponseBody(resp, respObj); err != nil { + if err := normalizeResponseBody(resp, &respObj); err != nil { return nil, err } response = append(response, respObj) @@ -67,7 +67,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect body: body, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } - if err := normalizeResponseBody(redirectResp, respObj); err != nil { + if err := normalizeResponseBody(redirectResp, &respObj); err != nil { return nil, err } response = append(response, respObj) @@ -77,7 +77,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect } // normalizeResponseBody performs normalization on the http response object. -func normalizeResponseBody(resp *http.Response, response redirectedResponse) error { +func normalizeResponseBody(resp *http.Response, response *redirectedResponse) error { var err error // net/http doesn't automatically decompress the response body if an // encoding has been specified by the user in the request so in case we have to From 93060ec0e30eb8ef78acbf5704bf8f62d701f70a Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 9 Nov 2021 08:50:18 +0100 Subject: [PATCH 108/196] adding tests --- v2/pkg/protocols/http/raw/raw_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index 0e7cee9b2..e661d3807 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -75,3 +75,13 @@ Connection: close`, "https://test.com/test/", true) require.Nil(t, err, "could not parse unsafe request") require.Contains(t, string(request.UnsafeRawBytes), "GET /test/manager/html", "Could not parse unsafe method request path correctly") } + +func TestTryFillCustomHeaders(t *testing.T) { + testValue := "GET /manager/html HTTP/1.1\r\nHost: Test\r\n" + expected := "GET /test/manager/html HTTP/1.1\r\nHost: Test\r\ntest: test\r\n" + request, err := Parse(testValue, "https://test.com/test/", true) + require.Nil(t, err, "could not parse unsafe request") + err = request.TryFillCustomHeaders([]string{"test: test"}) + require.Nil(t, err, "could not add custom headers") + require.Equal(t, expected, string(request.UnsafeRawBytes), "actual value and expected value are different") +} From f2c81ee9a2def6d103161652f2d6181b6a32e33f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 9 Nov 2021 16:11:11 +0530 Subject: [PATCH 109/196] fixed unsafe test --- integration_tests/http/raw-unsafe-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/http/raw-unsafe-request.yaml b/integration_tests/http/raw-unsafe-request.yaml index 0a84b9157..e7c45c983 100644 --- a/integration_tests/http/raw-unsafe-request.yaml +++ b/integration_tests/http/raw-unsafe-request.yaml @@ -7,7 +7,7 @@ info: requests: - raw: - - | + - |+ GET / HTTP/1.1 Host: Content-Length: 4 From 9b364080ea506d0cd3d96f14062c2f47ee839746 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 9 Nov 2021 17:55:42 +0530 Subject: [PATCH 110/196] Fixed some edge cases with status codes in redirect chains --- integration_tests/http/get-redirects-chain-headers.yaml | 7 ++++++- v2/cmd/integration-test/integration-test.go | 2 +- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/utils.go | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/integration_tests/http/get-redirects-chain-headers.yaml b/integration_tests/http/get-redirects-chain-headers.yaml index c4446fecd..512073018 100644 --- a/integration_tests/http/get-redirects-chain-headers.yaml +++ b/integration_tests/http/get-redirects-chain-headers.yaml @@ -11,8 +11,13 @@ requests: - "{{BaseURL}}" redirects: true max-redirects: 3 + matchers-condition: and matchers: - type: word part: header words: - - "TestRedirectHeaderMatch" \ No newline at end of file + - "TestRedirectHeaderMatch" + + - type: status + status: + - 302 \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 94e6c2159..72a81a678 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -51,5 +51,5 @@ func main() { } func errIncorrectResultsCount(results []string) error { - return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t")) + return fmt.Errorf("incorrect number of results \n\t%s", strings.Join(results, "\n\t")) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 16be32a61..9b5b02a63 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -445,7 +445,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } finalEvent := make(output.InternalEvent) - outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 14088eed9..5eae57efe 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -22,6 +22,7 @@ type redirectedResponse struct { headers []byte body []byte fullResponse []byte + resp *http.Response } // dumpResponseWithRedirectChain dumps a http response with the @@ -41,6 +42,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect respObj := redirectedResponse{ headers: respData, body: body, + resp: resp, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } if err := normalizeResponseBody(resp, &respObj); err != nil { @@ -65,6 +67,7 @@ func dumpResponseWithRedirectChain(resp *http.Response, body []byte) ([]redirect respObj := redirectedResponse{ headers: respData, body: body, + resp: redirectResp, fullResponse: bytes.Join([][]byte{respData, body}, []byte{}), } if err := normalizeResponseBody(redirectResp, &respObj); err != nil { From 4b169fcae1f264a9b895793aaa9cdd02a404877e Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 10 Nov 2021 15:09:40 +0530 Subject: [PATCH 111/196] mod update --- v2/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e1080d2aa..15980fdd6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -64,8 +64,6 @@ require ( moul.io/http2curl v1.0.0 ) -require github.com/weppos/publicsuffix-go v0.15.0 - require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect From 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:00:03 -0600 Subject: [PATCH 112/196] feat: Checking socks5 proxy before launching a scan #1001 (#1225) * Proxy validation and list input support Co-authored-by: Sajad Parra Co-authored-by: sandeep --- README.md | 20 +-- README_CN.md | 3 +- v2/cmd/nuclei/main.go | 6 +- v2/internal/runner/options.go | 25 +--- v2/internal/runner/proxy.go | 123 ++++++++++++++++++ v2/internal/testutils/testutils.go | 3 +- v2/pkg/protocols/headless/engine/engine.go | 4 +- .../protocols/headless/engine/http_client.go | 13 +- .../http/httpclientpool/clientpool.go | 45 +++---- v2/pkg/types/proxy.go | 15 +++ v2/pkg/types/types.go | 6 +- 11 files changed, 184 insertions(+), 79 deletions(-) create mode 100644 v2/internal/runner/proxy.go create mode 100644 v2/pkg/types/proxy.go diff --git a/README.md b/README.md index ca1988816..e4626a05a 100644 --- a/README.md +++ b/README.md @@ -162,16 +162,16 @@ HEADLESS: -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: - -debug show all requests and responses - -debug-req show all sent requests - -debug-resp show all received responses - -proxy, -proxy-url string URL of the HTTP proxy server - -proxy-socks-url string URL of the SOCKS proxy server - -tlog, -trace-log string file to write sent requests trace log - -version show nuclei version - -v, -verbose show verbose output - -vv display extra verbose information - -tv, -templates-version shows the version of the installed nuclei-templates + -debug show all requests and responses + -debug-req show all sent requests + -debug-resp show all received responses + -p, -proxy string[] List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) + -tlog, -trace-log string file to write sent requests trace log + -elog, -error-log string file to write sent requests error log + -version show nuclei version + -v, -verbose show verbose output + -vv display templates loaded for scan + -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: -update update nuclei engine to the latest released version diff --git a/README_CN.md b/README_CN.md index 04138acd2..0c4aa69c0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,8 +121,7 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| +|proxy|输入代理地址|nuclei -proxy ./proxy.txt| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f42041579..ac176732b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -141,11 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"), flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - - /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc - TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), - flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)"), flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 3866b744a..c8376f76a 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "errors" - "net/url" "os" "path/filepath" "strings" @@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) - // Show the user the banner showBanner() @@ -89,15 +87,10 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - - if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { + //loading the proxy server list from file or cli and test the connectivity + if err := loadProxyServers(options); err != nil { return err } - - if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { - return err - } - if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -114,19 +107,6 @@ func validateOptions(options *types.Options) error { return nil } -func validateProxyURL(proxyURL, message string) error { - if proxyURL != "" && !isValidURL(proxyURL) { - return errors.New(message) - } - - return nil -} - -func isValidURL(urlString string) bool { - _, err := url.Parse(urlString) - return err == nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -172,7 +152,6 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) - for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go new file mode 100644 index 000000000..e254472a7 --- /dev/null +++ b/v2/internal/runner/proxy.go @@ -0,0 +1,123 @@ +package runner + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var proxyURLList []url.URL + +// loadProxyServers load list of proxy servers from file or comma seperated +func loadProxyServers(options *types.Options) error { + if len(options.Proxy) == 0 { + return nil + } + for _, p := range options.Proxy { + if proxyURL, err := validateProxyURL(p); err == nil { + proxyURLList = append(proxyURLList, proxyURL) + } else if fileutil.FileExists(p) { + file, err := os.Open(p) + if err != nil { + return fmt.Errorf("could not open proxy file: %s", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + proxy := scanner.Text() + if strings.TrimSpace(proxy) == "" { + continue + } + if proxyURL, err := validateProxyURL(proxy); err != nil { + return err + } else { + proxyURLList = append(proxyURLList, proxyURL) + } + } + } else { + return fmt.Errorf("invalid proxy file or URL provided for %s", p) + } + } + return processProxyList(options) +} + +func processProxyList(options *types.Options) error { + if len(proxyURLList) == 0 { + return fmt.Errorf("could not find any valid proxy") + } else { + done := make(chan bool) + exitCounter := make(chan bool) + counter := 0 + for _, url := range proxyURLList { + go runProxyConnectivity(url, options, done, exitCounter) + } + for { + select { + case <-done: + { + close(done) + return nil + } + case <-exitCounter: + { + if counter += 1; counter == len(proxyURLList) { + return errors.New("no reachable proxy found") + } + } + } + } + } +} + +func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { + if err := testProxyConnection(proxyURL, options.Timeout); err == nil { + if types.ProxyURL == "" && types.ProxySocksURL == "" { + assignProxyURL(proxyURL, options) + done <- true + } + } + exitCounter <- true +} + +func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { + timeout := time.Duration(timeoutDelay) * time.Second + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + if err != nil { + return err + } + return nil +} + +func assignProxyURL(proxyURL url.URL, options *types.Options) { + os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) + if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + types.ProxyURL = proxyURL.String() + types.ProxySocksURL = "" + gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) + } else if proxyURL.Scheme == types.SOCKS5 { + types.ProxyURL = "" + types.ProxySocksURL = proxyURL.String() + gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) + } +} + +func validateProxyURL(proxy string) (url.URL, error) { + if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { + return *url, nil + } + return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") +} + +//isSupportedProtocol checks given protocols are supported +func isSupportedProtocol(value string) bool { + return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index 79e426422..a26a1a1cf 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - ProxyURL: "", - ProxySocksURL: "", + Proxy: []string{}, TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda4..a42595801 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if options.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) + if types.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 4ae14cf1e..20b7840a9 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -4,13 +4,14 @@ import ( "context" "crypto/tls" "fmt" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" "net/url" "time" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" "golang.org/x/net/proxy" @@ -36,15 +37,13 @@ func newhttpClient(options *types.Options) *http.Client { MaxConnsPerHost: 500, TLSClientConfig: tlsConfig, } - - if options.ProxyURL != "" { - if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } - } else if options.ProxySocksURL != "" { + } else if types.ProxySocksURL != "" { var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) + socksURL, proxyErr := url.Parse(types.ProxySocksURL) if proxyErr == nil { proxyAuth = &proxy.Auth{} proxyAuth.User = socksURL.User.Username() diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 2b7dc0245..2817b4592 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "fmt" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" "net" "net/http" "net/http/cookiejar" @@ -14,6 +13,8 @@ import ( "sync" "time" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "github.com/pkg/errors" "golang.org/x/net/proxy" "golang.org/x/net/publicsuffix" @@ -129,9 +130,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) + if types.ProxyURL != "" { + proxyURL, err = url.Parse(types.ProxyURL) } if err != nil { return nil, err @@ -179,27 +179,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl TLSClientConfig: tlsConfig, DisableKeepAlives: disableKeepAlives, } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) - if proxyErr == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) + // Attempts to overwrite the dial function with the socks proxied version + if proxyURL.Scheme == types.SOCKS5 { + var proxyAuth *proxy.Auth = &proxy.Auth{} + proxyAuth.User = proxyURL.User.Username() + proxyAuth.Password, _ = proxyURL.User.Password() + + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } else { + transport.Proxy = http.ProxyURL(proxyURL) + } } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go new file mode 100644 index 000000000..a45b4eca6 --- /dev/null +++ b/v2/pkg/types/proxy.go @@ -0,0 +1,15 @@ +package types + +const ( + HTTP_PROXY_ENV = "HTTP_PROXY" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" +) + +var ( + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string +) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index a40e460a9..bb8e659f8 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -58,10 +58,8 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string + // List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input) + Proxy goflags.NormalizedStringSlice // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests From 56b7d78fedb2eb946e3e7a8c095388f34ef41efe Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 10 Nov 2021 18:04:53 +0100 Subject: [PATCH 113/196] Fixing http test using local http mock server --- .../offlinehttp/read_response_test.go | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index 4a75d7263..f47b345dc 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -1,12 +1,15 @@ package offlinehttp import ( + "fmt" "io/ioutil" "net/http" + "net/http/httptest" "net/http/httputil" "testing" "time" + "github.com/julienschmidt/httprouter" "github.com/stretchr/testify/require" ) @@ -154,11 +157,30 @@ Server: Google Frontend } t.Run("test-live-response-with-content-length", func(t *testing.T) { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Server", "Google Frontend") + fmt.Fprintf(w, "%s", ` + + + Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ + `) + })) + ts = httptest.NewServer(router) + defer ts.Close() + client := &http.Client{ Timeout: 3 * time.Second, } - data, err := client.Get("https://golang.org/doc/install") + data, err := client.Get(ts.URL) require.Nil(t, err, "could not dial url") defer data.Body.Close() From 09b5fb11e8799e465aaa2b00397ddcb789f49e1f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Nov 2021 18:11:42 +0100 Subject: [PATCH 114/196] Better error handling for return value of helper http library (#1239) --- v2/pkg/protocols/http/request.go | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 9b5b02a63..3958d6c74 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -99,16 +99,16 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. mutex := &sync.Mutex{} for { generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() @@ -162,18 +162,17 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous mutex := &sync.Mutex{} for { generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } generatedHttpRequest.pipelinedClient = pipeClient - swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() @@ -216,17 +215,16 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err == io.EOF { - break + if err != nil { + if err == io.EOF { + break + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return err } if reqURL == "" { reqURL = generatedHttpRequest.URL() } - if err != nil { - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err - } - request.dynamicValues = generatedHttpRequest.dynamicValues // Check if hosts just keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { From ac75d9aa9bfe6f89576482dceb2febb0e5f69f5f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 10 Nov 2021 18:12:49 +0100 Subject: [PATCH 115/196] Improving error handling in client certificate library (#1237) --- v2/pkg/protocols/headless/engine/engine.go | 7 ++++++- v2/pkg/protocols/headless/engine/http_client.go | 10 +++++++--- v2/pkg/protocols/http/httpclientpool/clientpool.go | 5 ++++- v2/pkg/protocols/utils/utils.go | 12 ++++++------ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a42595801..c435bef40 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -88,7 +88,12 @@ func New(options *types.Options) (*Browser, error) { if customAgent == "" { customAgent = uarand.GetRandom() } - httpclient := newhttpClient(options) + + httpclient, err := newhttpClient(options) + if err != nil { + return nil, err + } + engine := &Browser{ tempDir: dataStore, customAgent: customAgent, diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 20b7840a9..1f40f98f7 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -18,7 +18,7 @@ import ( ) // newhttpClient creates a new http client for headless communication with a timeout -func newhttpClient(options *types.Options) *http.Client { +func newhttpClient(options *types.Options) (*http.Client, error) { dialer := protocolstate.Dialer // Set the base TLS configuration definition @@ -28,7 +28,11 @@ func newhttpClient(options *types.Options) *http.Client { } // Add the client certificate authentication to the request if it's configured - tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + var err error + tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + if err != nil { + return nil, err + } transport := &http.Transport{ DialContext: dialer.Dial, @@ -70,5 +74,5 @@ func newhttpClient(options *types.Options) *http.Client { }, } - return httpclient + return httpclient, nil } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 2817b4592..191ca3cf0 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -169,7 +169,10 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // Add the client certificate authentication to the request if it's configured - tlsConfig = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + tlsConfig, err = utils.AddConfiguredClientCertToRequest(tlsConfig, options) + if err != nil { + return nil, errors.Wrap(err, "could not create client certificate") + } transport := &http.Transport{ DialContext: Dialer.Dial, diff --git a/v2/pkg/protocols/utils/utils.go b/v2/pkg/protocols/utils/utils.go index 8d43fd478..d139f1287 100644 --- a/v2/pkg/protocols/utils/utils.go +++ b/v2/pkg/protocols/utils/utils.go @@ -3,13 +3,13 @@ package utils import ( "crypto/tls" "crypto/x509" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "io/ioutil" - "log" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // AddConfiguredClientCertToRequest adds the client certificate authentication to the tls.Config object and returns it -func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) *tls.Config { +func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Options) (*tls.Config, error) { // Build the TLS config with the client certificate if it has been configured with the appropriate options. // Only one of the options needs to be checked since the validation checks in main.go ensure that all three // files are set if any of the client certification configuration options are. @@ -17,18 +17,18 @@ func AddConfiguredClientCertToRequest(tlsConfig *tls.Config, options *types.Opti // Load the client certificate using the PEM encoded client certificate and the private key file cert, err := tls.LoadX509KeyPair(options.ClientCertFile, options.ClientKeyFile) if err != nil { - log.Fatal(err) + return nil, err } tlsConfig.Certificates = []tls.Certificate{cert} // Load the certificate authority PEM certificate into the TLS configuration caCert, err := ioutil.ReadFile(options.ClientCAFile) if err != nil { - log.Fatal(err) + return nil, err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig.RootCAs = caCertPool } - return tlsConfig + return tlsConfig, nil } From 1d74f6ada2cdf035789e41c416f1214786241c25 Mon Sep 17 00:00:00 2001 From: sullo Date: Wed, 10 Nov 2021 12:55:22 -0500 Subject: [PATCH 116/196] Typo (#1242) --- v2/pkg/protocols/dns/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 0cf54988e..d6be2f7df 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// Match matches a generic data response again a given matcher +// Match matches a generic data response against a given matcher func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := request.getMatchPart(matcher.Part, data) if !ok { From 90a0502b60d52e15faf8971975824964b5784709 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 11 Nov 2021 17:30:25 +0530 Subject: [PATCH 117/196] Misc fixes to default part --- v2/pkg/operators/extractors/compile.go | 5 ----- v2/pkg/operators/matchers/compile.go | 2 +- v2/pkg/protocols/http/operators.go | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 07306e207..567716675 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -42,11 +42,6 @@ func (e *Extractor) CompileExtractors() error { e.jsonCompiled = append(e.jsonCompiled, compiled) } - // Set up the part of the request to match, if any. - if e.Part == "" { - e.Part = "body" - } - if e.CaseInsensitive { if e.Type != "kval" { return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 63b3edf58..7c7c5555e 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -31,7 +31,7 @@ func (m *Matcher) CompileMatchers() error { } // By default, match on body if user hasn't provided any specific items if m.Part == "" { - m.Part = "body" + m.Part = "response" } // Compile the regexes diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 8d7c4b72a..868fd0118 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -76,6 +76,9 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto // getMatchPart returns the match part honoring "all" matchers + others. func (request *Request) getMatchPart(part string, data output.InternalEvent) (string, bool) { + if part == "" { + part = "body" + } if part == "header" { part = "all_headers" } From cdb08e08793f6f5d41de30ebf0f18b072e79ffbc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 12 Nov 2021 04:44:02 +0530 Subject: [PATCH 118/196] misc fixes to websocket --- v2/pkg/protocols/websocket/websocket.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index cdc40cc0c..47a87a78d 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -172,7 +172,11 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam payloadValues["Hostname"] = parsed.Host payloadValues["Host"] = parsed.Hostname() payloadValues["Scheme"] = parsed.Scheme - payloadValues["Path"] = parsed.Path + requestPath := parsed.Path + if values := parsed.Query(); len(values) > 0 { + requestPath = requestPath + "?" + values.Encode() + } + payloadValues["Path"] = requestPath requestOptions := request.options for key, value := range request.Headers { From dc0c568cb8a8d8fd58539f437d3d660e331812b8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 11 Nov 2021 23:16:47 +0000 Subject: [PATCH 119/196] Auto Generate Syntax Docs + JSONSchema [Thu Nov 11 23:16:47 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 353 +++++++++++++++++++++++++++++- nuclei-jsonschema.json | 165 +++++++++++++- v2/pkg/templates/templates_doc.go | 233 +++++++++++++++++--- 3 files changed, 707 insertions(+), 44 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 3d16bda71..d87ac8904 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -219,6 +219,32 @@ Headless contains the headless request to make in the template.

+ssl []ssl.Request + +
+
+ +SSL contains the SSL request to make in the template. + +
+ +
+ +
+ +websocket []websocket.Request + +
+
+ +Websocket contains the Websocket request to make in the template. + +
+ +
+ +
+ workflows []workflows.WorkflowTemplate
@@ -829,7 +855,7 @@ in a combined manner allowing multirequest based matchers.
-attack string +attack generators.AttackTypeHolder
@@ -1238,6 +1264,10 @@ Appears in: - headless.Request.matchers +- ssl.Request.matchers + +- websocket.Request.matchers +
@@ -1594,6 +1624,10 @@ Appears in: - headless.Request.extractors +- ssl.Request.extractors + +- websocket.Request.extractors +
@@ -1877,6 +1911,22 @@ Valid values: +## generators.AttackTypeHolder +AttackTypeHolder is used to hold internal type of the protocol + +Appears in: + + +- http.Request.attack + +- network.Request.attack + +- websocket.Request.attack + + + + + ## dns.Request Request contains a DNS protocol request to be made from a template @@ -1998,7 +2048,7 @@ name: '{{FQDN}}'
-Type is the type of DNS request to make. +RequestType is the type of DNS request to make. Valid values: @@ -2358,7 +2408,7 @@ host:
-attack string +attack generators.AttackTypeHolder
@@ -2852,6 +2902,303 @@ Valid values: +## ssl.Request +Request is a request for the SSL protocol + +Appears in: + + +- Template.ssl + + + +
+ +
+ +matchers []matchers.Matcher + +
+
+ +Matchers contains the detection mechanism for the request to identify +whether the request was successful by doing pattern matching +on request/responses. + +Multiple matchers can be combined with `matcher-condition` flag +which accepts either `and` or `or` as argument. + +
+ +
+ +
+ +extractors []extractors.Extractor + +
+
+ +Extractors contains the extraction mechanism for the request to identify +and extract parts of the response. + +
+ +
+ +
+ +matchers-condition string + +
+
+ +MatchersCondition is the condition between the matchers. Default is OR. + + +Valid values: + + + - and + + - or +
+ +
+ +
+ +address string + +
+
+ +Address contains address for the request + +
+ +
+ + + + + +## websocket.Request +Request is a request for the Websocket protocol + +Appears in: + + +- Template.websocket + + + +
+ +
+ +matchers []matchers.Matcher + +
+
+ +Matchers contains the detection mechanism for the request to identify +whether the request was successful by doing pattern matching +on request/responses. + +Multiple matchers can be combined with `matcher-condition` flag +which accepts either `and` or `or` as argument. + +
+ +
+ +
+ +extractors []extractors.Extractor + +
+
+ +Extractors contains the extraction mechanism for the request to identify +and extract parts of the response. + +
+ +
+ +
+ +matchers-condition string + +
+
+ +MatchersCondition is the condition between the matchers. Default is OR. + + +Valid values: + + + - and + + - or +
+ +
+ +
+ +address string + +
+
+ +Address contains address for the request + +
+ +
+ +
+ +inputs []websocket.Input + +
+
+ +Inputs contains inputs for the websocket protocol + +
+ +
+ +
+ +headers map[string]string + +
+
+ +Headers contains headers for the request. + +
+ +
+ +
+ +attack generators.AttackTypeHolder + +
+
+ +Attack is the type of payload combinations to perform. + +Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +permutations and combinations for all payloads. + + +Valid values: + + + - sniper + + - pitchfork + + - clusterbomb +
+ +
+ +
+ +payloads map[string]interface{} + +
+
+ +Payloads contains any payloads for the current request. + +Payloads support both key-values combinations where a list +of payloads is provided, or optionally a single file can also +be provided as payload which will be read on run-time. + +
+ +
+ + + + + +## websocket.Input + +Appears in: + + +- websocket.Request.inputs + + + +
+ +
+ +data string + +
+
+ +Data is the data to send as the input. + +It supports DSL Helper Functions as well as normal expressions. + + + +Examples: + + +```yaml +data: TEST +``` + +```yaml +data: hex_decode('50494e47') +``` + + +
+ +
+ +
+ +name string + +
+
+ +Name is the optional name of the data read to provide matching on. + + + +Examples: + + +```yaml +name: prefix +``` + + +
+ +
+ + + + + ## workflows.WorkflowTemplate Appears in: diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 4f22c3863..517876ea4 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -308,6 +308,16 @@ "additionalProperties": false, "type": "object" }, + "generators.AttackTypeHolder": { + "enum": [ + "batteringram", + "pitchfork", + "clusterbomb" + ], + "type": "string", + "title": "type of the attack", + "description": "Type of the attack" + }, "dns.Request": { "properties": { "matchers": { @@ -615,12 +625,8 @@ "description": "Optional name for the HTTP Request" }, "attack": { - "enum": [ - "batteringram", - "pitchfork", - "clusterbomb" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/generators.AttackTypeHolder", "title": "attack is the payload combination", "description": "Attack is the type of payload combinations to perform" }, @@ -786,12 +792,7 @@ "description": "Host to send network requests to" }, "attack": { - "enum": [ - "batteringram", - "pitchfork", - "clusterbomb" - ], - "type": "string", + "$ref": "#/definitions/generators.AttackTypeHolder", "title": "attack is the payload combination", "description": "Attack is the type of payload combinations to perform" }, @@ -853,6 +854,128 @@ "additionalProperties": false, "type": "object" }, + "ssl.Request": { + "properties": { + "matchers": { + "items": { + "$ref": "#/definitions/matchers.Matcher" + }, + "type": "array", + "title": "matchers to run on response", + "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + }, + "extractors": { + "items": { + "$ref": "#/definitions/extractors.Extractor" + }, + "type": "array", + "title": "extractors to run on response", + "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + }, + "matchers-condition": { + "enum": [ + "and", + "or" + ], + "type": "string", + "title": "condition between the matchers", + "description": "Conditions between the matchers" + }, + "address": { + "type": "string", + "title": "address for the ssl request", + "description": "Address contains address for the request" + } + }, + "additionalProperties": false, + "type": "object" + }, + "websocket.Input": { + "properties": { + "data": { + "type": "string", + "title": "data to send as input", + "description": "Data is the data to send as the input" + }, + "name": { + "type": "string", + "title": "optional name for data read", + "description": "Optional name of the data read to provide matching on" + } + }, + "additionalProperties": false, + "type": "object" + }, + "websocket.Request": { + "properties": { + "matchers": { + "items": { + "$ref": "#/definitions/matchers.Matcher" + }, + "type": "array", + "title": "matchers to run on response", + "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + }, + "extractors": { + "items": { + "$ref": "#/definitions/extractors.Extractor" + }, + "type": "array", + "title": "extractors to run on response", + "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + }, + "matchers-condition": { + "enum": [ + "and", + "or" + ], + "type": "string", + "title": "condition between the matchers", + "description": "Conditions between the matchers" + }, + "address": { + "type": "string", + "title": "address for the websocket request", + "description": "Address contains address for the request" + }, + "inputs": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/websocket.Input" + }, + "type": "array", + "title": "inputs for the websocket request", + "description": "Inputs contains any input/output for the current request" + }, + "headers": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object", + "title": "headers contains the request headers", + "description": "Headers contains headers for the request" + }, + "attack": { + "$ref": "#/definitions/generators.AttackTypeHolder", + "title": "attack is the payload combination", + "description": "Attack is the type of payload combinations to perform" + }, + "payloads": { + "patternProperties": { + ".*": { + "additionalProperties": true + } + }, + "type": "object", + "title": "payloads for the webosocket request", + "description": "Payloads contains any payloads for the current request" + } + }, + "additionalProperties": false, + "type": "object" + }, "templates.Template": { "required": [ "id", @@ -919,6 +1042,24 @@ "title": "headless requests to make", "description": "Headless requests to make for the template" }, + "ssl": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/ssl.Request" + }, + "type": "array", + "title": "ssl requests to make", + "description": "SSL requests to make for the template" + }, + "websocket": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/websocket.Request" + }, + "type": "array", + "title": "websocket requests to make", + "description": "Websocket requests to make for the template" + }, "workflows": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index ca416b3b8..b24ccdaa5 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -9,29 +9,33 @@ import ( ) var ( - TemplateDoc encoder.Doc - MODELInfoDoc encoder.Doc - STRINGSLICEStringSliceDoc encoder.Doc - SEVERITYHolderDoc encoder.Doc - MODELClassificationDoc encoder.Doc - HTTPRequestDoc encoder.Doc - MATCHERSMatcherDoc encoder.Doc - EXTRACTORSExtractorDoc encoder.Doc - DNSRequestDoc encoder.Doc - FILERequestDoc encoder.Doc - NETWORKRequestDoc encoder.Doc - NETWORKInputDoc encoder.Doc - HEADLESSRequestDoc encoder.Doc - ENGINEActionDoc encoder.Doc - WORKFLOWSWorkflowTemplateDoc encoder.Doc - WORKFLOWSMatcherDoc encoder.Doc + TemplateDoc encoder.Doc + MODELInfoDoc encoder.Doc + STRINGSLICEStringSliceDoc encoder.Doc + SEVERITYHolderDoc encoder.Doc + MODELClassificationDoc encoder.Doc + HTTPRequestDoc encoder.Doc + MATCHERSMatcherDoc encoder.Doc + EXTRACTORSExtractorDoc encoder.Doc + GENERATORSAttackTypeHolderDoc encoder.Doc + DNSRequestDoc encoder.Doc + FILERequestDoc encoder.Doc + NETWORKRequestDoc encoder.Doc + NETWORKInputDoc encoder.Doc + HEADLESSRequestDoc encoder.Doc + ENGINEActionDoc encoder.Doc + SSLRequestDoc encoder.Doc + WEBSOCKETRequestDoc encoder.Doc + WEBSOCKETInputDoc encoder.Doc + WORKFLOWSWorkflowTemplateDoc encoder.Doc + WORKFLOWSMatcherDoc encoder.Doc ) func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 9) + TemplateDoc.Fields = make([]encoder.Doc, 11) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -79,16 +83,26 @@ func init() { TemplateDoc.Fields[6].Note = "" TemplateDoc.Fields[6].Description = "Headless contains the headless request to make in the template." TemplateDoc.Fields[6].Comments[encoder.LineComment] = "Headless contains the headless request to make in the template." - TemplateDoc.Fields[7].Name = "workflows" - TemplateDoc.Fields[7].Type = "[]workflows.WorkflowTemplate" + TemplateDoc.Fields[7].Name = "ssl" + TemplateDoc.Fields[7].Type = "[]ssl.Request" TemplateDoc.Fields[7].Note = "" - TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[8].Name = "self-contained" - TemplateDoc.Fields[8].Type = "bool" + TemplateDoc.Fields[7].Description = "SSL contains the SSL request to make in the template." + TemplateDoc.Fields[7].Comments[encoder.LineComment] = "SSL contains the SSL request to make in the template." + TemplateDoc.Fields[8].Name = "websocket" + TemplateDoc.Fields[8].Type = "[]websocket.Request" TemplateDoc.Fields[8].Note = "" - TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained" - TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template." + TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template." + TemplateDoc.Fields[9].Name = "workflows" + TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate" + TemplateDoc.Fields[9].Note = "" + TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[10].Name = "self-contained" + TemplateDoc.Fields[10].Type = "bool" + TemplateDoc.Fields[10].Note = "" + TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -320,7 +334,7 @@ func init() { HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multirequest based matchers." HTTPRequestDoc.Fields[6].Comments[encoder.LineComment] = "Name is the optional name of the request." HTTPRequestDoc.Fields[7].Name = "attack" - HTTPRequestDoc.Fields[7].Type = "string" + HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" HTTPRequestDoc.Fields[7].Note = "" HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." @@ -472,6 +486,14 @@ func init() { TypeName: "headless.Request", FieldName: "matchers", }, + { + TypeName: "ssl.Request", + FieldName: "matchers", + }, + { + TypeName: "websocket.Request", + FieldName: "matchers", + }, } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" @@ -609,6 +631,14 @@ func init() { TypeName: "headless.Request", FieldName: "extractors", }, + { + TypeName: "ssl.Request", + FieldName: "extractors", + }, + { + TypeName: "websocket.Request", + FieldName: "extractors", + }, } EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11) EXTRACTORSExtractorDoc.Fields[0].Name = "name" @@ -697,6 +727,25 @@ func init() { "true", } + GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" + GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" + GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" + GENERATORSAttackTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "attack", + }, + { + TypeName: "network.Request", + FieldName: "attack", + }, + { + TypeName: "websocket.Request", + FieldName: "attack", + }, + } + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) + DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" DNSRequestDoc.Description = "Request contains a DNS protocol request to be made from a template" @@ -743,8 +792,8 @@ func init() { DNSRequestDoc.Fields[5].Name = "type" DNSRequestDoc.Fields[5].Type = "string" DNSRequestDoc.Fields[5].Note = "" - DNSRequestDoc.Fields[5].Description = "Type is the type of DNS request to make." - DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "Type is the type of DNS request to make." + DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make." + DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make." DNSRequestDoc.Fields[5].Values = []string{ "A", "NS", @@ -875,7 +924,7 @@ func init() { NETWORKRequestDoc.Fields[1].AddExample("", []string{"{{Hostname}}"}) NETWORKRequestDoc.Fields[2].Name = "attack" - NETWORKRequestDoc.Fields[2].Type = "string" + NETWORKRequestDoc.Fields[2].Type = "generators.AttackTypeHolder" NETWORKRequestDoc.Fields[2].Note = "" NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." @@ -1065,6 +1114,128 @@ func init() { "sleep", } + SSLRequestDoc.Type = "ssl.Request" + SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol" + SSLRequestDoc.Description = "Request is a request for the SSL protocol" + SSLRequestDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "ssl", + }, + } + SSLRequestDoc.Fields = make([]encoder.Doc, 4) + SSLRequestDoc.Fields[0].Name = "matchers" + SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" + SSLRequestDoc.Fields[0].Note = "" + SSLRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + SSLRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + SSLRequestDoc.Fields[1].Name = "extractors" + SSLRequestDoc.Fields[1].Type = "[]extractors.Extractor" + SSLRequestDoc.Fields[1].Note = "" + SSLRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + SSLRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + SSLRequestDoc.Fields[2].Name = "matchers-condition" + SSLRequestDoc.Fields[2].Type = "string" + SSLRequestDoc.Fields[2].Note = "" + SSLRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR." + SSLRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + SSLRequestDoc.Fields[2].Values = []string{ + "and", + "or", + } + SSLRequestDoc.Fields[3].Name = "address" + SSLRequestDoc.Fields[3].Type = "string" + SSLRequestDoc.Fields[3].Note = "" + SSLRequestDoc.Fields[3].Description = "Address contains address for the request" + SSLRequestDoc.Fields[3].Comments[encoder.LineComment] = "Address contains address for the request" + + WEBSOCKETRequestDoc.Type = "websocket.Request" + WEBSOCKETRequestDoc.Comments[encoder.LineComment] = " Request is a request for the Websocket protocol" + WEBSOCKETRequestDoc.Description = "Request is a request for the Websocket protocol" + WEBSOCKETRequestDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "websocket", + }, + } + WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) + WEBSOCKETRequestDoc.Fields[0].Name = "matchers" + WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" + WEBSOCKETRequestDoc.Fields[0].Note = "" + WEBSOCKETRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + WEBSOCKETRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + WEBSOCKETRequestDoc.Fields[1].Name = "extractors" + WEBSOCKETRequestDoc.Fields[1].Type = "[]extractors.Extractor" + WEBSOCKETRequestDoc.Fields[1].Note = "" + WEBSOCKETRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + WEBSOCKETRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + WEBSOCKETRequestDoc.Fields[2].Name = "matchers-condition" + WEBSOCKETRequestDoc.Fields[2].Type = "string" + WEBSOCKETRequestDoc.Fields[2].Note = "" + WEBSOCKETRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR." + WEBSOCKETRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + WEBSOCKETRequestDoc.Fields[2].Values = []string{ + "and", + "or", + } + WEBSOCKETRequestDoc.Fields[3].Name = "address" + WEBSOCKETRequestDoc.Fields[3].Type = "string" + WEBSOCKETRequestDoc.Fields[3].Note = "" + WEBSOCKETRequestDoc.Fields[3].Description = "Address contains address for the request" + WEBSOCKETRequestDoc.Fields[3].Comments[encoder.LineComment] = "Address contains address for the request" + WEBSOCKETRequestDoc.Fields[4].Name = "inputs" + WEBSOCKETRequestDoc.Fields[4].Type = "[]websocket.Input" + WEBSOCKETRequestDoc.Fields[4].Note = "" + WEBSOCKETRequestDoc.Fields[4].Description = "Inputs contains inputs for the websocket protocol" + WEBSOCKETRequestDoc.Fields[4].Comments[encoder.LineComment] = "Inputs contains inputs for the websocket protocol" + WEBSOCKETRequestDoc.Fields[5].Name = "headers" + WEBSOCKETRequestDoc.Fields[5].Type = "map[string]string" + WEBSOCKETRequestDoc.Fields[5].Note = "" + WEBSOCKETRequestDoc.Fields[5].Description = "Headers contains headers for the request." + WEBSOCKETRequestDoc.Fields[5].Comments[encoder.LineComment] = "Headers contains headers for the request." + WEBSOCKETRequestDoc.Fields[6].Name = "attack" + WEBSOCKETRequestDoc.Fields[6].Type = "generators.AttackTypeHolder" + WEBSOCKETRequestDoc.Fields[6].Note = "" + WEBSOCKETRequestDoc.Fields[6].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + WEBSOCKETRequestDoc.Fields[6].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." + WEBSOCKETRequestDoc.Fields[6].Values = []string{ + "sniper", + "pitchfork", + "clusterbomb", + } + WEBSOCKETRequestDoc.Fields[7].Name = "payloads" + WEBSOCKETRequestDoc.Fields[7].Type = "map[string]interface{}" + WEBSOCKETRequestDoc.Fields[7].Note = "" + WEBSOCKETRequestDoc.Fields[7].Description = "Payloads contains any payloads for the current request.\n\nPayloads support both key-values combinations where a list\nof payloads is provided, or optionally a single file can also\nbe provided as payload which will be read on run-time." + WEBSOCKETRequestDoc.Fields[7].Comments[encoder.LineComment] = "Payloads contains any payloads for the current request." + + WEBSOCKETInputDoc.Type = "websocket.Input" + WEBSOCKETInputDoc.Comments[encoder.LineComment] = "" + WEBSOCKETInputDoc.Description = "" + WEBSOCKETInputDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "websocket.Request", + FieldName: "inputs", + }, + } + WEBSOCKETInputDoc.Fields = make([]encoder.Doc, 2) + WEBSOCKETInputDoc.Fields[0].Name = "data" + WEBSOCKETInputDoc.Fields[0].Type = "string" + WEBSOCKETInputDoc.Fields[0].Note = "" + WEBSOCKETInputDoc.Fields[0].Description = "Data is the data to send as the input.\n\nIt supports DSL Helper Functions as well as normal expressions." + WEBSOCKETInputDoc.Fields[0].Comments[encoder.LineComment] = "Data is the data to send as the input." + + WEBSOCKETInputDoc.Fields[0].AddExample("", "TEST") + + WEBSOCKETInputDoc.Fields[0].AddExample("", "hex_decode('50494e47')") + WEBSOCKETInputDoc.Fields[1].Name = "name" + WEBSOCKETInputDoc.Fields[1].Type = "string" + WEBSOCKETInputDoc.Fields[1].Note = "" + WEBSOCKETInputDoc.Fields[1].Description = "Name is the optional name of the data read to provide matching on." + WEBSOCKETInputDoc.Fields[1].Comments[encoder.LineComment] = "Name is the optional name of the data read to provide matching on." + + WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix") + WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate" WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = "" WORKFLOWSWorkflowTemplateDoc.Description = "" @@ -1144,12 +1315,16 @@ func GetTemplateDoc() *encoder.FileDoc { &HTTPRequestDoc, &MATCHERSMatcherDoc, &EXTRACTORSExtractorDoc, + &GENERATORSAttackTypeHolderDoc, &DNSRequestDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, + &SSLRequestDoc, + &WEBSOCKETRequestDoc, + &WEBSOCKETInputDoc, &WORKFLOWSWorkflowTemplateDoc, &WORKFLOWSMatcherDoc, }, From 7af5121de898dda227b4906445dcaa25b315a585 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 12 Nov 2021 19:06:59 +0530 Subject: [PATCH 120/196] Added high level nuclei architecture overview (#1177) * Added high level nuclei architecture overview --- DESIGN.md | 604 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 604 insertions(+) create mode 100644 DESIGN.md diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 000000000..f9875b0b6 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,604 @@ +# Nuclei Architecture Document + +A brief overview of Nuclei Engine architecture. This document will be kept updated as the engine progresses. + +## pkg/templates + +### Template + +Template is the basic unit of input to the engine which describes the requests to be made, matching to be done, data to extract, etc. + +The template structure is described here. Template level attributes are defined here as well as convenience methods to validate, parse and compile templates creating executers. + +Any attributes etc required for the template, engine or requests to function are also set here. + +Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc on the paths provided are also done here. + +`Parse` function is the main entry point which returns a template for a `filePath` and `executorOptions`. It compiles all the requests for the templates, all the workflows, as well as any self-contained request etc. It also caches the templates in an in-memory cache. + +### Preprocessors + +Preprocessors are also applied here which can do things at template level. They get data of the template which they can alter at will on runtime. This is used in the engine to do random string generation. + +Custom processor can be used if they satisfy the following interface. + +```go +type Preprocessor interface { + Process(data []byte) []byte +} +``` + +## pkg/model + +Model package implements Information structure for Nuclei Templates. `Info` contains all major metadata information for the template. `Classification` structure can also be used to provide additional context to vulnerability data. + +It also specifies a `WorkflowLoader` interface that is used during workflow loading in template compilation stage. + +```go +type WorkflowLoader interface { + GetTemplatePathsByTags(tags []string) []string + GetTemplatePaths(templatesList []string, noValidate bool) []string +} +``` + +## pkg/protocols + +Protocols package implements all the request protocols supported by Nuclei. This includes http, dns, network, headless and file requests as of now. + +### Request + +It exposes a `Request` interface that is implemented by all the request protocols supported. + +```go +// Request is an interface implemented any protocol based request generator. +type Request interface { + Compile(options *ExecuterOptions) error + Requests() int + GetID() string + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) + Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent + MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + GetCompiledOperators() []*operators.Operators +} +``` + +Many of these methods are similar across protocols while some are very protocol specific. + +A brief overview of the methods is provided below - + +- **Compile** - Compiles the request with provided options. +- **Requests** - Returns total requests made. +- **GetID** - Returns any ID for the request +- **Match** - Used to perform matching for patterns using matchers +- **Extract** - Used to perform extraction for patterns using extractors +- **ExecuteWithResults** - Request execution function for input. +- **MakeResultEventItem** - Creates a single result event for the intermediate `InternalWrappedEvent` output structure. +- **MakeResultEvent** - Returns a slice of results based on an `InternalWrappedEvent` internal output event. +- **GetCompiledOperators** - Returns the compiled operators for the request. + +`MakeDefaultResultEvent` function can be used as a default for `MakeResultEvent` function when no protocol-specific features need to be implemented for result generation. + +For reference protocol requests implementations, one can look at the below packages - + +1. [pkg/protocols/http](./v2/pkg/protocols/http) +2. [pkg/protocols/dns](./v2/pkg/protocols/dns) +3. [pkg/protocols/network](./v2/pkg/protocols/network) + +### Executer + +All these different requests interfaces are converted to an Executer which is also an interface defined in `pkg/protocols` which is used during final execution of the template. + +```go +// Executer is an interface implemented any protocol based request executer. +type Executer interface { + Compile() error + Requests() int + Execute(input string) (bool, error) + ExecuteWithResults(input string, callback OutputEventCallback) error +} +``` + +The `ExecuteWithResults` function accepts a callback, which gets provided with results during execution in form of `*output.InternalWrappedEvent` structure. + +The default executer is provided in `pkg/protocols/common/executer` . It takes a list of Requests and relevant `ExecuterOptions` and implements the Executer interface required for template execution. The executer during Template compilation process is created from this package and used as-is. + +A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/protocols/common/clusterer` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. + +For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. + +With this basic premise set, we can now start exploring the current runner implementation which will also walk us through the architecture of nuclei. + +## internal/runner + +### Template loading + +The first process after all CLI specific initialisation is the loading of template/workflow paths that the user wants to run. This is done by the packages described below. + +#### pkg/catalog + +This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template directory as well as the current user directory. + +The syntax is very versatile and can include filenames, glob patterns, directories, absolute paths, and relative-paths. + + + +Next step is the initialisation of the reporting modules which is handled in `pkg/reporting`. + +#### pkg/reporting + +Reporting module contains exporters and trackers as well as a module for deduplication and a module for result formatting. + +Exporters and Trackers are interfaces defined in pkg/reporting. + +```go +// Tracker is an interface implemented by an issue tracker +type Tracker interface { + CreateIssue(event *output.ResultEvent) error +} + +// Exporter is an interface implemented by an issue exporter +type Exporter interface { + Close() error + Export(event *output.ResultEvent) error +} +``` + +Exporters include `Elasticsearch`, `markdown`, `sarif` . Trackers include `GitHub` , `Gitlab` and `Jira`. + +Each exporter and trackers implement their own configuration in YAML format and are very modular in nature, so adding new ones is easy. + + + +After reading all the inputs from various sources and initialisation other miscellaneous options, the next bit is the output writing which is done using `pkg/output` module. + +#### pkg/output + +Output package implements the output writing functionality for Nuclei. + +Output Writer implements the Writer interface which is called each time a result is found for nuclei. + +```go +// Writer is an interface which writes output to somewhere for nuclei events. +type Writer interface { + Close() + Colorizer() aurora.Aurora + Write(*ResultEvent) error + Request(templateID, url, requestType string, err error) +} +``` + +ResultEvent structure is passed to the Nuclei Output Writer which contains the entire detail of a found result. Various intermediary types like `InternalWrappedEvent` and `InternalEvent` are used throughout nuclei protocols and matchers to describe results in various stages of execution. + + + + Interactsh is also initialised if it is not explicitly disabled. + +#### pkg/protocols/common/interactsh + +Interactsh module is used to provide automatic Out of Band vulnerability identification in Nuclei. + +It uses two LRU caches, one for storing interactions for request URLs and one for storing requests for interaction URL. These both caches are used to correlated requests received to the Interactsh OOB server and Nuclei Instance. [Interactsh Client](https://github.com/projectdiscovery/interactsh/pkg/client) package does most of the heavy lifting of this module. + +Polling for interactions and server registration only starts when a template uses the interactsh module and is executed by nuclei. After that no registration is required for the entire run. + + + +### RunEnumeration + +Next we arrive in the `RunEnumeration` function of the runner. + +`HostErrorsCache` is initialised which is used throughout the run of Nuclei enumeration to keep track of errors per host and skip further requests if the errors are greater than the provided threshold. The functionality for the error tracking cache is defined in [hosterrorscache.go](https://github.com/projectdiscovery/nuclei/blob/master/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go) and is pretty simplistic in nature. + +Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `v2/pkg/parsers/workflow_loader.go` + +The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc to return compiled `Templates` and `Workflows`. + +#### pkg/catalog/loader + +First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to removed the excluded template/workflows paths. + +`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. + +`Parse` function performs compilation of all the requests in a template + creates Executers from them returning a runnable Template/Workflow structure. + +Clustering module comes in next whose job is to cluster identical HTTP GET requests together (as a lot of the templates perform the same get requests many times, it's a good way to save many requests on large scans with lots of templates). + +### pkg/operators + +Operators package implements all of the matching and extracting logic of Nuclei. + +```go +// Operators contains the operators that can be applied on protocols +type Operators struct { + Matchers []*matchers.Matcher + Extractors []*extractors.Extractor + MatchersCondition string +} +``` + +A protocol only needs to embed the `operators.Operators` type shown above and it can utilise all the matching/extracting functionality of nuclei. + +```go +// MatchFunc performs matching operation for a matcher on model and returns true or false. +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) + +// ExtractFunc performs extracting operation for an extractor on model and returns true or false. +type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + +// Execute executes the operators on data and returns a result structure +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) +``` + +The core of this process is the Execute function which takes an input dictionary as well as a Match and Extract function and return a `Result` structure which is used later during nuclei execution to check for results. + +```go +// Result is a result structure created from operators running on data. +type Result struct { + Matched bool + Extracted bool + Matches map[string][]string + Extracts map[string][]string + OutputExtracts []string + DynamicValues map[string]interface{} + PayloadValues map[string]interface{} +} +``` + +The internal logics for matching and extracting for things like words, regexes, jq, paths, etc is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. + + +### Template Execution + +`pkg/core` provides the engine mechanism which runs the templates/workflows on inputs. It exposes an `Execute` function which does the task of execution while also doing template clustring. The clustering can also be disbled optionally by the user. + +An example of using the core engine is provided below. + +```go +engine := core.New(r.options) +engine.SetExecuterOptions(executerOpts) +results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true) +``` + +### Using Nuclei From Go Code + +An example of using Nuclei From Go Code to run templates on targets is provided below. + +```go +package main + +import ( + "fmt" + "log" + "os" + "path" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/core" + "github.com/projectdiscovery/nuclei/v2/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "go.uber.org/ratelimit" +) + +func main() { + cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount) + defer cache.Close() + + mockProgress := &testutils.MockProgressClient{} + reportingClient, _ := reporting.New(&reporting.Options{}, "") + defer reportingClient.Close() + + outputWriter := testutils.NewMockOutputWriter() + outputWriter.WriteCallback = func(event *output.ResultEvent) { + fmt.Printf("Got Result: %v\n", event) + } + + defaultOpts := types.DefaultOptions() + protocolstate.Init(defaultOpts) + protocolinit.Init(defaultOpts) + + defaultOpts.Templates = goflags.StringSlice{"dns/cname-service-detection.yaml"} + defaultOpts.ExcludeTags = config.ReadIgnoreFile().Tags + + interactOpts := interactsh.NewDefaultOptions(outputWriter, reportingClient, mockProgress) + interactClient, err := interactsh.New(interactOpts) + if err != nil { + log.Fatalf("Could not create interact client: %s\n", err) + } + defer interactClient.Close() + + home, _ := os.UserHomeDir() + catalog := catalog.New(path.Join(home, "nuclei-templates")) + executerOpts := protocols.ExecuterOptions{ + Output: outputWriter, + Options: defaultOpts, + Progress: mockProgress, + Catalog: catalog, + IssuesClient: reportingClient, + RateLimiter: ratelimit.New(150), + Interactsh: interactClient, + HostErrorsCache: cache, + Colorizer: aurora.NewAurora(true), + } + engine := core.New(defaultOpts) + engine.SetExecuterOptions(executerOpts) + + workflowLoader, err := parsers.NewLoader(&executerOpts) + if err != nil { + log.Fatalf("Could not create workflow loader: %s\n", err) + } + executerOpts.WorkflowLoader = workflowLoader + + store, err := loader.New(loader.NewConfig(defaultOpts, catalog, executerOpts)) + if err != nil { + log.Fatalf("Could not create loader client: %s\n", err) + } + store.Load() + + input := &inputs.SimpleInputProvider{Inputs: []string{"docs.hackerone.com"}} + _ = engine.Execute(store.Templates(), input) +} +``` + +### Adding a New Protocol + +Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc are implemented in form of protocol requests. + +A protocol must implement the `Protocol` and `Request` interfaces described above in `pkg/protocols`. We'll take the example of an existing protocol implementation - websocket for this short reference around Nuclei internals. + +The code for the websocket protocol is contained in `pkg/protocols/others/websocket`. + +Below a high level skeleton of the websocket implementation is provided with all the important parts present. + +```go +package websocket + +// Request is a request for the Websocket protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Address contains address for the request + Address string `yaml:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` + + // declarations here +} + +// Compile compiles the request generators preparing any requests possible. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + + // request compilation here as well as client creation + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (r *Request) Requests() int { + if r.generator != nil { + return r.generator.NewIterator().Total() + } + return 1 +} + +// GetID returns the ID for the request if any. +func (r *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + // payloads init here + if err := r.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + return err + } + return nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + header := http.Header{} + + // make the actual request here after setting all options + + event := eventcreator.CreateEventWithAdditionalOptions(r, data, r.options.Options.Debug || r.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues + }) + if r.options.Options.Debug || r.options.Options.DebugResponse { + responseOutput := responseBuilder.String() + gologger.Debug().Msgf("[%s] Dumped Websocket response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, r.options.Options.NoColor)) + } + + callback(event) + return nil +} + +func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(r.options.TemplateID), + TemplatePath: types.ToString(r.options.TemplatePath), + // ... setting more values for result event + } + return data +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(r, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (r *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{r.CompiledOperators} +} + +// Type returns the type of the protocol request +func (r *Request) Type() templateTypes.ProtocolType { + return templateTypes.WebsocketProtocol +} +``` + +Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. + +`eventcreator` package offers `CreateEventWithAdditionalOptions` function which can be used to create result events after doing request execution. + +Step by step description of how to add a new protocol to Nuclei - + +1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with less number of options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. + +2. Add the protocol request structure to the `Template` structure fields. This is done in `pkg/templates/templates.go` with the corresponding import line. + +```go + +import ( + ... + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket" +) + +// Template is a YAML input file which defines all the requests and +// other metadata for a template. +type Template struct { + ... + // description: | + // Websocket contains the Websocket request to make in the template. + RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` + ... +} +``` + +Also add the protocol case to the `Type` function as well as the `TemplateTypes` array in the same `templates.go` file. + +```go +// TemplateTypes is a list of accepted template types +var TemplateTypes = []string{ + ... + "websocket", +} + +// Type returns the type of the template +func (t *Template) Type() templateTypes.ProtocolType { + ... + case len(t.RequestsWebsocket) > 0: + return templateTypes.WebsocketProtocol + default: + return "" + } +} +``` + +3. Add the protocol request to the `Requests` function and `compileProtocolRequests` function in the `compile.go` file in same directory. + +```go + +// Requests returns the total request count for the template +func (template *Template) Requests() int { + return len(template.RequestsDNS) + + ... + len(template.RequestsSSL) + + len(template.RequestsWebsocket) +} + + +// compileProtocolRequests compiles all the protocol requests for the template +func (template *Template) compileProtocolRequests(options protocols.ExecuterOptions) error { + ... + + case len(template.RequestsWebsocket) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket) + } + template.Executer = executer.NewExecuter(requests, &options) + return nil +} +``` + +That's it, you've added a new protocol to Nuclei. The next good step would be to write integration tests which are described in `integration-tests` and `cmd/integration-tests` directories. + +## Project Structure + +- [v2/pkg/reporting](./v2/pkg/reporting) - Reporting modules for nuclei. +- [v2/pkg/reporting/exporters/sarif](./v2/pkg/reporting/exporters/sarif) - Sarif Result Exporter +- [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter +- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/e) - Elasticsearch Result Exporter +- [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results +- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter +- [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter +- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - Github Issue Tracker Exporter +- [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions +- [v2/pkg/parsers](./v2/pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading. +- [v2/pkg/types](./v2/pkg/types) - Contains CLI options as well as misc helper functions. +- [v2/pkg/progress](./v2/pkg/progress) - Progress tracking +- [v2/pkg/operators](./v2/pkg/operators) - Operators for Nuclei +- [v2/pkg/operators/common/dsl](./v2/pkg/operators/common/dsl) - DSL functions for Nuclei YAML Syntax +- [v2/pkg/operators/matchers](./v2/pkg/operators/matchers) - Matchers implementation +- [v2/pkg/operators/extractors](./v2/pkg/operators/extractors) - Extractors implementation +- [v2/pkg/catalog](./v2/pkg/catalog) - Template loading from disk helpers +- [v2/pkg/catalog/config](./v2/pkg/catalog/config) - Internal configuration management +- [v2/pkg/catalog/loader](./v2/pkg/catalog/loader) - Implements loading and validation of templates and workflows. +- [v2/pkg/catalog/loader/filter](./v2/pkg/catalog/loader/filter) - Filter filters templates based on tags and paths +- [v2/pkg/output](./v2/pkg/output) - Output module for nuclei +- [v2/pkg/workflows](./v2/pkg/workflows) - Workflow execution logic + declarations +- [v2/pkg/utils](./v2/pkg/utils) - Utility functions +- [v2/pkg/model](./v2/pkg/model) - Template Info + misc +- [v2/pkg/templates](./v2/pkg/templates) - Templates core starting point +- [v2/pkg/templates/cache](./v2/pkg/templates/cache) - Templates cache +- [v2/pkg/protocols](./v2/pkg/protocol) - Protocol Specification +- [v2/pkg/protocols/file](./v2/pkg/protocols/file) - File protocol +- [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol +- [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support +- [v2/pkg/protocols/common/interactsh](./v2/pkg/protocols/common/interactsh) - Interactsh integration +- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) +- [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer +- [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers +- [v2/pkg/protocols/common/clusterer](./v2/pkg/protocols/common/clusterer) - HTTP Request Clustering Implementation +- [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator +- [v2/pkg/protocols/common/helpers/responsehighlighter](./v2/pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter +- [v2/pkg/protocols/common/helpers/deserialization](./v2/pkg/protocols/common/helpers/deserialization) - Deserialization helper functions +- [v2/pkg/protocols/common/hosterrorscache](./v2/pkg/protocols/common/hosterrorscache) - Host errors cache for tracking erroring hosts +- [v2/pkg/protocols/offlinehttp](./v2/pkg/protocols/offlinehttp) - Offline http protocol +- [v2/pkg/protocols/http](./v2/pkg/protocols/http) - HTTP protocol +- [v2/pkg/protocols/http/race](./v2/pkg/protocols/http/race) - HTTP Race Module +- [v2/pkg/protocols/http/raw](./v2/pkg/protocols/http/raw) - HTTP Raw Request Support +- [v2/pkg/protocols/headless](./v2/pkg/protocols/headless) - Headless Module +- [v2/pkg/protocols/headless/engine](./v2/pkg/protocols/headless/engine) - Internal Headless implementation +- [v2/pkg/protocols/dns](./v2/pkg/protocols/dns) - DNS protocol +- [v2/pkg/projectfile](./v2/pkg/projectfile) - Project File Implementation + +### Notes + +1. The matching as well as interim output functionality is a bit complex, we should simplify it a bit as well. From cc33366acb69c55c4fb7049a1867227fac376eb0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 14 Nov 2021 15:30:28 +0530 Subject: [PATCH 121/196] fixed a conditon with workflow type switch --- v2/pkg/templates/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 39e0e447d..d34cd1ae1 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -108,7 +108,7 @@ func (t *Template) Type() types.ProtocolType { return types.HeadlessProtocol case len(t.RequestsNetwork) > 0: return types.NetworkProtocol - case t.CompiledWorkflow != nil: + case len(t.Workflow.Workflows) > 0: return types.WorkflowProtocol case len(t.RequestsSSL) > 0: return types.SSLProtocol From 50a816fce9e28587b48b73b250e7aa6dbe02c14f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 16 Nov 2021 20:02:39 +0530 Subject: [PATCH 122/196] Fixed a crash with uninitialized interactsh client (#1251) * Fixed a crash with uninitialized interactsh client --- v2/internal/runner/runner.go | 29 +++++++++-------- v2/pkg/operators/matchers/compile.go | 4 --- .../protocols/common/interactsh/interactsh.go | 6 +++- v2/pkg/protocols/http/build_request.go | 31 +++++++++++++------ v2/pkg/protocols/network/request.go | 7 ++++- .../protocols/offlinehttp/operators_test.go | 1 + 6 files changed, 48 insertions(+), 30 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a5981fad..5e71b76fe 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -144,22 +144,21 @@ func New(options *types.Options) (*Runner, error) { } } - if !options.NoInteractsh { - opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) - opts.Debug = runner.options.Debug - opts.ServerURL = options.InteractshURL - opts.Authorization = options.InteractshToken - opts.CacheSize = int64(options.InteractionsCacheSize) - opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second - opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second - opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + opts := interactsh.NewDefaultOptions(runner.output, runner.issuesClient, runner.progress) + opts.Debug = runner.options.Debug + opts.ServerURL = options.InteractshURL + opts.Authorization = options.InteractshToken + opts.CacheSize = int64(options.InteractionsCacheSize) + opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second + opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second + opts.NoInteractsh = runner.options.NoInteractsh - interactshClient, err := interactsh.New(opts) - if err != nil { - gologger.Error().Msgf("Could not create interactsh client: %s", err) - } else { - runner.interactsh = interactshClient - } + interactshClient, err := interactsh.New(opts) + if err != nil { + gologger.Error().Msgf("Could not create interactsh client: %s", err) + } else { + runner.interactsh = interactshClient } if options.RateLimitMinute > 0 { diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 7c7c5555e..d4937614c 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -29,10 +29,6 @@ func (m *Matcher) CompileMatchers() error { if !ok { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } - // By default, match on body if user hasn't provided any specific items - if m.Part == "" { - m.Part = "response" - } // Compile the regexes for _, regex := range m.Regex { diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index c11f5ad18..7c3e02f82 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -72,6 +72,8 @@ type Options struct { Progress progress.Progress // Debug specifies whether debugging output should be shown for interactsh-client Debug bool + + NoInteractsh bool } const defaultMaxInteractionsCount = 5000 @@ -118,6 +120,9 @@ func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progre } func (c *Client) firstTimeInitializeClient() error { + if c.options.NoInteractsh { + return nil // do not init if disabled + } interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, Token: c.options.Authorization, @@ -221,7 +226,6 @@ func (c *Client) Close() bool { // It accepts data to replace as well as the URL to replace placeholders // with generated uniquely for each request. func (c *Client) ReplaceMarkers(data string, interactshURLs []string) (string, []string) { - for strings.Contains(data, interactshURLMarker) { url := c.URL() interactshURLs = append(interactshURLs, url) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 4b4ac9f21..6b9b01f18 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -64,10 +64,15 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) - - for payloadName, payloadValue := range payloads { - payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + for payloadName, payloadValue := range payloads { + payloads[payloadName], r.interactshURLs = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), r.interactshURLs) + } + } else { + for payloadName, payloadValue := range payloads { + payloads[payloadName] = types.ToString(payloadValue) + } } parsed, err := url.Parse(baseURL) @@ -171,7 +176,9 @@ func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { // MakeHTTPRequestFromModel creates a *http.Request from a request template func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + } // Combine the template payloads along with base // request values. @@ -204,7 +211,9 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { - data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + if r.options.Interactsh != nil { + data, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(data, r.interactshURLs) + } return r.handleRawWithPayloads(ctx, data, baseURL, values, payloads) } @@ -268,7 +277,9 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { // Set the header values requested for header, value := range r.request.Headers { - value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) + if r.options.Interactsh != nil { + value, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(value, r.interactshURLs) + } value, err := expressions.Evaluate(value, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") @@ -286,8 +297,10 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte // Check if the user requested a request body if r.request.Body != "" { - var body string - body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) + body := r.request.Body + if r.options.Interactsh != nil { + body, r.interactshURLs = r.options.Interactsh.ReplaceMarkers(r.request.Body, r.interactshURLs) + } body, err := expressions.Evaluate(body, values) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 98c65ee4b..f5229d123 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -140,7 +140,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input case "hex": data, err = hex.DecodeString(input.Data) default: - input.Data, interactshURLs = request.options.Interactsh.ReplaceMarkers(input.Data, []string{}) data = []byte(input.Data) } if err != nil { @@ -150,6 +149,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input } reqBuilder.Grow(len(input.Data)) + if request.options.Interactsh != nil { + var transformedData string + transformedData, interactshURLs = request.options.Interactsh.ReplaceMarkers(string(data), []string{}) + data = []byte(transformedData) + } + finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), dataErr) diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 8669d8b58..05fe9468f 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -154,6 +154,7 @@ func TestHTTPOperatorExtract(t *testing.T) { extractor := &extractors.Extractor{ Type: "kval", KVal: []string{"test-header"}, + Part: "header", } err = extractor.CompileExtractors() require.Nil(t, err, "could not compile kval extractor") From eaa0d370656ff6bb1a7e40556245660d9f895fd4 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 17 Nov 2021 02:04:27 +0530 Subject: [PATCH 123/196] feat #1092: Validate binary values + precompile them as well (#1213) * Added validation for binary values + precompile them * Changed name of the binary matcher field Co-authored-by: sandeep --- v2/pkg/operators/matchers/compile.go | 9 +++++++++ v2/pkg/operators/matchers/match.go | 19 ++++--------------- v2/pkg/operators/matchers/matchers.go | 1 + 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index d4937614c..0efebb1aa 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -39,6 +39,15 @@ func (m *Matcher) CompileMatchers() error { m.regexCompiled = append(m.regexCompiled, compiled) } + // Compile and validate binary Values in matcher + for _, value := range m.Binary { + if decoded, err := hex.DecodeString(value); err != nil { + return fmt.Errorf("could not hex decode binary: %s", value) + } else { + m.binaryDecoded = append(m.binaryDecoded, string(decoded)) + } + } + // Compile the dsl expressions for _, expr := range m.DSL { compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index df71a1442..84601be6e 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -1,10 +1,8 @@ package matchers import ( - "encoding/hex" "strings" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -120,17 +118,8 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { func (m *Matcher) MatchBinary(corpus string) (bool, []string) { var matchedBinary []string // Iterate over all the words accepted as valid - for i, binary := range m.Binary { - // Continue if the word doesn't match - hexa, err := hex.DecodeString(binary) - if err != nil { - gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) - if m.condition == ANDCondition { - return false, []string{} - } - continue - } - if !strings.Contains(corpus, string(hexa)) { + for i, binary := range m.binaryDecoded { + if !strings.Contains(corpus, binary) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { @@ -142,10 +131,10 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true, []string{string(hexa)} + return true, []string{binary} } - matchedBinary = append(matchedBinary, string(hexa)) + matchedBinary = append(matchedBinary, binary) // If we are at the end of the words, return with true if len(m.Binary)-1 == i { diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 594a6fc1d..4c0fd1fee 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -115,6 +115,7 @@ type Matcher struct { // cached data for the compiled matcher condition ConditionType matcherType MatcherType + binaryDecoded []string regexCompiled []*regexp.Regexp dslCompiled []*govaluate.EvaluableExpression } From 4ff90c1373ca81f8800decd2086504b7e49a8e9e Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:47:34 -0600 Subject: [PATCH 124/196] struct to enum changes for Dns Type (#1245) * struct to enum changes for Dns Type --- v2/pkg/protocols/dns/dns.go | 4 +- v2/pkg/protocols/dns/dns_test.go | 2 +- v2/pkg/protocols/dns/dns_types.go | 110 +++++++++++++++++++++ v2/pkg/protocols/dns/operators_test.go | 8 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/templates/templates_doc_examples.go | 2 +- 6 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 v2/pkg/protocols/dns/dns_types.go diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index a42df7ef1..a630762f1 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -43,7 +43,7 @@ type Request struct { // - "MX" // - "TXT" // - "AAAA" - RequestType string `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` + RequestType DNSRequestTypeHolder `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. // @@ -111,7 +111,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } request.class = classToInt(request.Class) request.options = options - request.question = questionTypeToInt(request.RequestType) + request.question = questionTypeToInt(request.RequestType.String()) return nil } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index b0f8dddfe..f44ba2205 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -27,7 +27,7 @@ func TestDNSCompileMake(t *testing.T) { testutils.Init(options) const templateID = "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go new file mode 100644 index 000000000..e1354fedf --- /dev/null +++ b/v2/pkg/protocols/dns/dns_types.go @@ -0,0 +1,110 @@ +package dns + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// DNSRequestType is the type of the method specified +type DNSRequestType int + +const ( + A DNSRequestType = iota + 1 + NS + DS + CNAME + SOA + PTR + MX + TXT + AAAA + //limit + limit +) + +// DNSRequestTypeMapping is a table for conversion of method from string. +var DNSRequestTypeMapping = map[DNSRequestType]string{ + A: "A", + NS: "NS", + DS: "DS", + CNAME: "CNAME", + SOA: "SOA", + PTR: "PTR", + MX: "MX", + TXT: "TXT", + AAAA: "AAAA", +} + +// GetSupportedDNSRequestTypes returns list of supported types +func GetSupportedDNSRequestTypes() []DNSRequestType { + var result []DNSRequestType + for index := DNSRequestType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toDNSRequestTypes(valueToMap string) (DNSRequestType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range DNSRequestTypeMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid DNS request type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToUpper(value)) +} + +func (t DNSRequestType) String() string { + return DNSRequestTypeMapping[t] +} + +// DNSRequestTypeHolder is used to hold internal type of the DNS type +type DNSRequestTypeHolder struct { + DNSRequestType DNSRequestType +} + +func (holder DNSRequestTypeHolder) String() string { + return holder.DNSRequestType.String() +} + +func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of DNS request to make", + Description: "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA", + } + for _, types := range GetSupportedDNSRequestTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *DNSRequestTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toDNSRequestTypes(marshalledTypes) + if err != nil { + return err + } + + holder.DNSRequestType = computedType + return nil +} + +func (holder *DNSRequestTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.DNSRequestType.String()) +} + +func (holder DNSRequestTypeHolder) MarshalYAML() (interface{}, error) { + return holder.DNSRequestType.String(), nil +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index e5618f610..bb87b52eb 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -23,7 +23,7 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -55,7 +55,7 @@ func TestDNSOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -166,7 +166,7 @@ func TestDNSOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, @@ -223,7 +223,7 @@ func TestDNSMakeResult(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index d2ed83994..38827f817 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -20,7 +20,7 @@ func TestDNSExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-dns" request := &Request{ - RequestType: "A", + RequestType: DNSRequestTypeHolder{DNSRequestType: A}, Class: "INET", Retries: 5, ID: templateID, diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index b755de290..e6655c6d9 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -38,7 +38,7 @@ var ( exampleNormalDNSRequest = &dns.Request{ Name: "{{FQDN}}", - RequestType: "CNAME", + RequestType: dns.DNSRequestTypeHolder{DNSRequestType: dns.CNAME}, Class: "inet", Retries: 2, Recursion: true, From 64a93a457051df381475ca9c00a3a508bbc55f68 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:48:47 -0600 Subject: [PATCH 125/196] enum changes for network type (#1233) * enum changes for network type --- v2/pkg/protocols/network/network.go | 6 +- .../protocols/network/network_input_types.go | 100 ++++++++++++++++++ v2/pkg/protocols/network/request.go | 4 +- v2/pkg/protocols/network/request_test.go | 2 +- 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 v2/pkg/protocols/network/network_input_types.go diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 83c9ac813..98037eed3 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -105,7 +105,7 @@ type Input struct { // values: // - "hex" // - "text" - Type string `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` + Type NetworkInputTypeHolder `yaml:"type,omitempty" jsonschema:"title=type is the type of input data,description=Type of input specified in data field,enum=hex,enum=text"` // description: | // Read is the number of bytes to read from socket. // @@ -153,7 +153,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } // Pre-compile any input dsl functions before executing the request. for _, input := range request.Inputs { - if input.Type != "" { + if input.Type.String() != "" { continue } if compiled, evalErr := expressions.Evaluate(input.Data, map[string]interface{}{}); evalErr == nil { @@ -167,7 +167,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { // check if inputs contains the payload var hasPayloadName bool for _, input := range request.Inputs { - if input.Type != "" { + if input.Type.String() != "" { continue } if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go new file mode 100644 index 000000000..f96c3e492 --- /dev/null +++ b/v2/pkg/protocols/network/network_input_types.go @@ -0,0 +1,100 @@ +package network + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// NetworkInputType is the type of the method specified +type NetworkInputType int + +const ( + hexType NetworkInputType = iota + 1 + textType + //limit + limit +) + +// NetworkInputMapping is a table for conversion of method from string. +var NetworkInputMapping = map[NetworkInputType]string{ + hexType: "hex", + textType: "text", +} + +// GetSupportedNetworkInputTypes returns list of supported types +func GetSupportedNetworkInputTypes() []NetworkInputType { + var result []NetworkInputType + for index := NetworkInputType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toNetworkInputTypes(valueToMap string) (NetworkInputType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range NetworkInputMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid network type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t NetworkInputType) String() string { + return NetworkInputMapping[t] +} + +// NetworkInputTypeHolder is used to hold internal type of the Network type +type NetworkInputTypeHolder struct { + NetworkInputType NetworkInputType +} + +func (holder NetworkInputTypeHolder) GetType() NetworkInputType { + return holder.NetworkInputType +} + +func (holder NetworkInputTypeHolder) String() string { + return holder.NetworkInputType.String() +} + +func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type is the type of input data", + Description: "description=Type of input specified in data field,enum=hex,enum=text", + } + for _, types := range GetSupportedNetworkInputTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *NetworkInputTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toNetworkInputTypes(marshalledTypes) + if err != nil { + return err + } + + holder.NetworkInputType = computedType + return nil +} + +func (holder *NetworkInputTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.NetworkInputType.String()) +} + +func (holder NetworkInputTypeHolder) MarshalYAML() (interface{}, error) { + return holder.NetworkInputType.String(), nil +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index f5229d123..ae63dc0d0 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -136,8 +136,8 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input for _, input := range request.Inputs { var data []byte - switch input.Type { - case "hex": + switch input.Type.GetType() { + case hexType: data, err = hex.DecodeString(input.Data) default: data = []byte(input.Data) diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 8b44091e2..0e8d194bd 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -91,7 +91,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { require.Equal(t, "

Example Domain

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") finalEvent = nil - request.Inputs[0].Type = "hex" + request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType} request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) t.Run("hex-to-string", func(t *testing.T) { From 25189c3ae79273b4c9ccc328fdee6c41704dc8c4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:50:07 +0000 Subject: [PATCH 126/196] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:50:07 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 4 +-- nuclei-jsonschema.json | 46 +++++++++++++++++++------------ v2/pkg/templates/templates_doc.go | 4 +-- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index d87ac8904..8ea798291 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2043,7 +2043,7 @@ name: '{{FQDN}}'
-type string +type DNSRequestTypeHolder
@@ -2609,7 +2609,7 @@ data: hex_decode('50494e47')
-type string +type NetworkInputTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 517876ea4..da66f3e9b 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -318,6 +318,22 @@ "title": "type of the attack", "description": "Type of the attack" }, + "dns.DNSRequestTypeHolder": { + "enum": [ + "A", + "NS", + "DS", + "CNAME", + "SOA", + "PTR", + "MX", + "TXT", + "AAAA" + ], + "type": "string", + "title": "type of DNS request to make", + "description": "Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA" + }, "dns.Request": { "properties": { "matchers": { @@ -356,18 +372,8 @@ "description": "Name is the Hostname to make DNS request for" }, "type": { - "enum": [ - "A", - "NS", - "DS", - "CNAME", - "SOA", - "PTR", - "MX", - "TXT", - "AAAA" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/dns.DNSRequestTypeHolder", "title": "type of dns request to make", "description": "Type is the type of DNS request to make" }, @@ -754,11 +760,8 @@ "description": "Data is the data to send as the input" }, "type": { - "enum": [ - "hex", - "text" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/network.NetworkInputTypeHolder", "title": "type is the type of input data", "description": "Type of input specified in data field" }, @@ -776,6 +779,15 @@ "additionalProperties": false, "type": "object" }, + "network.NetworkInputTypeHolder": { + "enum": [ + "hex", + "text" + ], + "type": "string", + "title": "type is the type of input data", + "description": "description=Type of input specified in data field,enum=hex,enum=text" + }, "network.Request": { "properties": { "id": { diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index b24ccdaa5..c7f1b016b 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -790,7 +790,7 @@ func init() { DNSRequestDoc.Fields[4].AddExample("", "{{FQDN}}") DNSRequestDoc.Fields[5].Name = "type" - DNSRequestDoc.Fields[5].Type = "string" + DNSRequestDoc.Fields[5].Type = "DNSRequestTypeHolder" DNSRequestDoc.Fields[5].Note = "" DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make." DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make." @@ -997,7 +997,7 @@ func init() { NETWORKInputDoc.Fields[0].AddExample("", "hex_decode('50494e47')") NETWORKInputDoc.Fields[1].Name = "type" - NETWORKInputDoc.Fields[1].Type = "string" + NETWORKInputDoc.Fields[1].Type = "NetworkInputTypeHolder" NETWORKInputDoc.Fields[1].Note = "" NETWORKInputDoc.Fields[1].Description = "Type is the type of input specified in `data` field.\n\nDefault value is text, but hex can be used for hex formatted data." NETWORKInputDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of input specified in `data` field." From d078b7238102f77c3ec296e8adc6a6027c58b476 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 07:50:21 -0600 Subject: [PATCH 127/196] enum changes for http method (#1234) * enum changes for http method --- SYNTAX-REFERENCE.md | 2 +- nuclei-jsonschema.json | 32 +++--- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/build_request_test.go | 6 +- v2/pkg/protocols/http/cluster_test.go | 4 +- v2/pkg/protocols/http/http.go | 4 +- v2/pkg/protocols/http/http_method_types.go | 112 ++++++++++++++++++++ v2/pkg/protocols/http/operators_test.go | 8 +- v2/pkg/templates/templates_doc.go | 2 +- v2/pkg/templates/templates_doc_examples.go | 2 +- 10 files changed, 146 insertions(+), 28 deletions(-) create mode 100644 v2/pkg/protocols/http/http_method_types.go diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 8ea798291..ed19bd9c0 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -880,7 +880,7 @@ Valid values:
-method string +method HTTPMethodTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index da66f3e9b..1d179d68a 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -576,6 +576,23 @@ "additionalProperties": false, "type": "object" }, + "http.HTTPMethodTypeHolder": { + "enum": [ + "GET", + "HEAD", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE" + ], + "type": "string", + "title": "method is the http request method", + "description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE" + }, "http.Request": { "properties": { "matchers": { @@ -637,19 +654,8 @@ "description": "Attack is the type of payload combinations to perform" }, "method": { - "enum": [ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PURGE" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/http.HTTPMethodTypeHolder", "title": "method is the http request method", "description": "Method is the HTTP Request Method" }, diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 6b9b01f18..8c8a2dbc8 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -191,7 +191,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st return nil, errors.Wrap(err, "could not evaluate helper expressions") } - method, err := expressions.Evaluate(r.request.Method, finalValues) + method, err := expressions.Evaluate(r.request.Method.String(), finalValues) if err != nil { return nil, errors.Wrap(err, "could not evaluate helper expressions") } diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 429676a37..a77e3c947 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -71,7 +71,7 @@ func TestMakeRequestFromModal(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}/login.php"}, - Method: "POST", + Method: HTTPMethodTypeHolder{MethodType: HTTPPost}, Body: "username=test&password=pass", Headers: map[string]string{ "Content-Type": "application/x-www-form-urlencoded", @@ -103,7 +103,7 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?query=example"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -211,7 +211,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}/?u=http://{{interactsh-url}}/&href=http://{{interactsh-url}}/&action=http://{{interactsh-url}}/&host={{interactsh-url}}"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, diff --git a/v2/pkg/protocols/http/cluster_test.go b/v2/pkg/protocols/http/cluster_test.go index 136b3feb9..a16db41ee 100644 --- a/v2/pkg/protocols/http/cluster_test.go +++ b/v2/pkg/protocols/http/cluster_test.go @@ -10,6 +10,6 @@ func TestCanCluster(t *testing.T) { req := &Request{Unsafe: true} require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request") - req = &Request{Path: []string{"{{BaseURL}}"}, Method: "GET"} - require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: "GET"}), "could not cluster GET request") + req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}} + require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index f85e49c2c..3fbe3a8ec 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -66,7 +66,7 @@ type Request struct { // - "TRACE" // - "PATCH" // - "PURGE" - Method string `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` + Method HTTPMethodTypeHolder `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` // description: | // Body is an optional parameter which contains HTTP Request body. // examples: @@ -242,7 +242,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { var hasPayloadName bool // search for markers in all request parts var inputs []string - inputs = append(inputs, request.Method, request.Body) + inputs = append(inputs, request.Method.String(), request.Body) inputs = append(inputs, request.Raw...) for k, v := range request.customHeaders { inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go new file mode 100644 index 000000000..987dde979 --- /dev/null +++ b/v2/pkg/protocols/http/http_method_types.go @@ -0,0 +1,112 @@ +package http + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// HTTPMethodType is the type of the method specified +type HTTPMethodType int + +const ( + HTTPGet HTTPMethodType = iota + 1 + HTTPHead + HTTPPost + HTTPPut + HTTPDelete + HTTPConnect + HTTPOptions + HTTPTrace + HTTPPatch + HTTPPurge + //limit + limit +) + +// HTTPMethodMapping is a table for conversion of method from string. +var HTTPMethodMapping = map[HTTPMethodType]string{ + HTTPGet: "GET", + HTTPHead: "HEAD", + HTTPPost: "POST", + HTTPPut: "PUT", + HTTPDelete: "DELETE", + HTTPConnect: "CONNECT", + HTTPOptions: "OPTIONS", + HTTPTrace: "TRACE", + HTTPPatch: "PATCH", + HTTPPurge: "PURGE", +} + +// GetSupportedHTTPMethodTypes returns list of supported types +func GetSupportedHTTPMethodTypes() []HTTPMethodType { + var result []HTTPMethodType + for index := HTTPMethodType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toHTTPMethodTypes(valueToMap string) (HTTPMethodType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range HTTPMethodMapping { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid HTTP method verb: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToUpper(value)) +} + +func (t HTTPMethodType) String() string { + return HTTPMethodMapping[t] +} + +// HTTPMethodTypeHolder is used to hold internal type of the HTTP Method +type HTTPMethodTypeHolder struct { + MethodType HTTPMethodType +} + +func (holder HTTPMethodTypeHolder) String() string { + return holder.MethodType.String() +} + +func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "method is the HTTP request method", + Description: "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE", + } + for _, types := range GetSupportedHTTPMethodTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *HTTPMethodTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toHTTPMethodTypes(marshalledTypes) + if err != nil { + return err + } + + holder.MethodType = computedType + return nil +} + +func (holder *HTTPMethodTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MethodType.String()) +} + +func (holder HTTPMethodTypeHolder) MarshalYAML() (interface{}, error) { + return holder.MethodType.String(), nil +} diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index bc7b888a2..7950bc64f 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -25,7 +25,7 @@ func TestResponseToDSLMap(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -55,7 +55,7 @@ func TestHTTPOperatorMatch(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -143,7 +143,7 @@ func TestHTTPOperatorExtract(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -257,7 +257,7 @@ func TestHTTPMakeResult(t *testing.T) { ID: templateID, Name: "testing", Path: []string{"{{BaseURL}}?test=1"}, - Method: "GET", + Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index c7f1b016b..52c5da75a 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -344,7 +344,7 @@ func init() { "clusterbomb", } HTTPRequestDoc.Fields[8].Name = "method" - HTTPRequestDoc.Fields[8].Type = "string" + HTTPRequestDoc.Fields[8].Type = "HTTPMethodTypeHolder" HTTPRequestDoc.Fields[8].Note = "" HTTPRequestDoc.Fields[8].Description = "Method is the HTTP Request Method." HTTPRequestDoc.Fields[8].Comments[encoder.LineComment] = "Method is the HTTP Request Method." diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index e6655c6d9..88d5e5e88 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -24,7 +24,7 @@ var ( Tags: stringslice.StringSlice{Value: "cve,cve2021,rce,ruby"}, } exampleNormalHTTPRequest = &http.Request{ - Method: "GET", + Method: http.HTTPMethodTypeHolder{MethodType: http.HTTPGet}, Path: []string{"{{BaseURL}}/.git/config"}, Operators: operators.Operators{ MatchersCondition: "and", From bef7bfc393fc27f16b275a8ba1e177868410ad5a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:51:52 +0000 Subject: [PATCH 128/196] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:51:52 UTC 2021] :robot: --- nuclei-jsonschema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 1d179d68a..9ace729ac 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -590,7 +590,7 @@ "PURGE" ], "type": "string", - "title": "method is the http request method", + "title": "method is the HTTP request method", "description": "Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE" }, "http.Request": { From 7f5f791e2342e5d4206d611be6d97f96f863235d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 18 Nov 2021 14:52:11 +0100 Subject: [PATCH 129/196] Adding dns trace support in dns templates (#1236) * Adding dns trace support in dns templates + minor refactoring --- v2/go.mod | 2 +- v2/go.sum | 4 +- v2/pkg/protocols/common/executer/executer.go | 2 - .../protocols/common/expressions/variables.go | 80 ++++++++++--------- v2/pkg/protocols/dns/dns.go | 34 +++++++- v2/pkg/protocols/dns/operators.go | 19 ++++- v2/pkg/protocols/dns/operators_test.go | 12 +-- v2/pkg/protocols/dns/request.go | 41 +++++++++- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/network/network.go | 2 +- 10 files changed, 145 insertions(+), 53 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 15980fdd6..c7d51d01e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -36,7 +36,7 @@ require ( github.com/projectdiscovery/interactsh v0.0.6 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 - github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a + github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/yamldoc-go v1.0.2 diff --git a/v2/go.sum b/v2/go.sum index afea1cbb5..c649ceb45 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -582,6 +582,7 @@ github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/d github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -618,8 +619,9 @@ github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohT github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= +github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 h1:Ooa5htghPkdyfpzy6Y5KLdyv4w8ePZWmfzFSPQlJStQ= +github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660/go.mod h1:UfszkO3x+GLKVOpXB7boddJKbwNCr+tMPSkfgCSNhl4= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index d9a285948..3aaf774cb 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -47,8 +47,6 @@ func (e *Executer) Execute(input string) (bool, error) { dynamicValues := make(map[string]interface{}) previous := make(map[string]interface{}) for _, req := range e.requests { - req := req - err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { ID := req.GetID() if ID != "" { diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 48ac546ed..26b566317 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -10,47 +10,55 @@ var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7 // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. -func ContainsUnresolvedVariables(data string) error { - matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return nil - } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables found: ") - - for i, match := range matches { - if len(match) < 2 { - continue +func ContainsUnresolvedVariables(items ...string) error { + for _, data := range items { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil } - errorString.WriteString(match[1]) - if i != len(matches)-1 { - errorString.WriteString(",") - } - } - errorMessage := errorString.String() - return errors.New(errorMessage) -} + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables found: ") -func ContainsVariablesWithNames(data string, names map[string]interface{}) error { - matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) - if len(matches) == 0 { - return nil - } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables with values found: ") - - for i, match := range matches { - if len(match) < 2 { - continue - } - matchName := match[1] - if _, ok := names[matchName]; !ok { - errorString.WriteString(matchName) + for i, match := range matches { + if len(match) < 2 { + continue + } + errorString.WriteString(match[1]) if i != len(matches)-1 { errorString.WriteString(",") } } + errorMessage := errorString.String() + return errors.New(errorMessage) } - errorMessage := errorString.String() - return errors.New(errorMessage) + + return nil +} + +func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error { + for _, data := range items { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables with values found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + matchName := match[1] + if _, ok := names[matchName]; !ok { + errorString.WriteString(matchName) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) + } + + return nil } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index a630762f1..7cde7fed4 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/retryabledns" @@ -62,6 +63,15 @@ type Request struct { // - name: Use a retry of 3 to 5 generally // value: 5 Retries int `yaml:"retries,omitempty" jsonschema:"title=retries for dns request,description=Retries is the number of retries for the DNS request"` + // description: | + // Trace performs a trace operation for the target. + Trace bool `yaml:"trace,omitempty" jsonschema:"title=trace operation,description=Trace performs a trace operation for the target."` + // description: | + // TraceMaxRecursion is the number of max recursion allowed for trace operations + // examples: + // - name: Use a retry of 100 to 150 generally + // value: 100 + TraceMaxRecursion int `yaml:"trace-max-recursion,omitempty" jsonschema:"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations"` CompiledOperators *operators.Operators `yaml:"-"` dnsClient *retryabledns.Client @@ -96,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { dnsClientOptions.Resolvers = request.Resolvers } // Create a dns client for the class - client, err := dnsclientpool.Get(options.Options, dnsClientOptions) + client, err := request.getDnsClient(options, nil) if err != nil { return errors.Wrap(err, "could not get dns client") } @@ -115,6 +125,28 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { return nil } +func (request *Request) getDnsClient(options *protocols.ExecuterOptions, metadata map[string]interface{}) (*retryabledns.Client, error) { + dnsClientOptions := &dnsclientpool.Configuration{ + Retries: request.Retries, + } + if len(request.Resolvers) > 0 { + if len(request.Resolvers) > 0 { + for _, resolver := range request.Resolvers { + if expressions.ContainsUnresolvedVariables(resolver) != nil { + var err error + resolver, err = expressions.Evaluate(resolver, metadata) + if err != nil { + return nil, errors.Wrap(err, "could not resolve resolvers expressions") + } + dnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver) + } + } + } + dnsClientOptions.Resolvers = request.Resolvers + } + return dnsclientpool.Get(options.Options, dnsClientOptions) +} + // Requests returns the total number of requests the YAML rule will perform func (request *Request) Requests() int { return 1 diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index d6be2f7df..12de136a8 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -2,6 +2,8 @@ package dns import ( "bytes" + "fmt" + "strings" "time" "github.com/miekg/dns" @@ -12,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryabledns" ) // Match matches a generic data response against a given matcher @@ -73,7 +76,7 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (in } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, tracedata *retryabledns.TraceData) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, @@ -87,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, + "trace": traceToString(tracedata, false), } } @@ -126,3 +130,16 @@ func questionToString(resourceRecords []dns.Question) string { } return buffer.String() } + +func traceToString(tracedata *retryabledns.TraceData, withSteps bool) string { + buffer := &bytes.Buffer{} + if tracedata != nil { + for i, dnsRecord := range tracedata.DNSData { + if withSteps { + buffer.WriteString(fmt.Sprintf("request %d to resolver %s:\n", i, strings.Join(dnsRecord.Resolver, ","))) + } + buffer.WriteString(dnsRecord.Raw) + } + } + return buffer.String() +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index bb87b52eb..1937660c9 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -44,8 +44,8 @@ func TestResponseToDSLMap(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 12, "could not get correct number of items in dsl map") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) + require.Len(t, event, 13, "could not get correct number of items in dsl map") require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode") } @@ -76,7 +76,7 @@ func TestDNSOperatorMatch(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ @@ -143,7 +143,7 @@ func TestDNSOperatorMatch(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "ONE.ONE.ONE.ONE."}}) - event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE") + event := request.responseToDSLMap(req, resp, "ONE.ONE.ONE.ONE", "ONE.ONE.ONE.ONE", nil) matcher := &matchers.Matcher{ Part: "raw", @@ -187,7 +187,7 @@ func TestDNSOperatorExtract(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ @@ -257,7 +257,7 @@ func TestDNSMakeResult(t *testing.T) { resp.Rcode = dns.RcodeSuccess resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") + event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 97cb9ca20..614159e2a 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/retryabledns" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) @@ -40,6 +41,14 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return errors.Wrap(err, "could not build request") } + dnsClient := request.dnsClient + if varErr := expressions.ContainsUnresolvedVariables(request.Resolvers...); varErr != nil { + if dnsClient, varErr = request.getDnsClient(request.options, metadata); varErr != nil { + gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) + return nil + } + } + requestString := compiledRequest.String() if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil { gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) @@ -51,7 +60,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // Send the request to the target servers - response, err := request.dnsClient.Do(compiledRequest) + response, err := dnsClient.Do(compiledRequest) if err != nil { request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) @@ -64,20 +73,33 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review request.options.Output.Request(request.options.TemplatePath, domain, request.Type().String(), err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) - outputEvent := request.responseToDSLMap(compiledRequest, response, input, input) + // perform trace if necessary + var tracedata *retryabledns.TraceData + if request.Trace { + tracedata, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) + if err != nil { + request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) + } + } + + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, tracedata) for k, v := range previous { outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + // TODO: dynamic values are not supported yet dumpResponse(event, request.options, response.String(), domain) + if request.Trace { + dumpTraceData(event, request.options, traceToString(tracedata, true), domain) + } callback(event) return nil } -func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response string, domain string) { +func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, response, domain string) { cliOptions := requestOptions.Options if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false @@ -90,6 +112,19 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. } } +func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, tracedata, domain string) { + cliOptions := requestOptions.Options + if cliOptions.Debug || cliOptions.DebugResponse { + hexDump := false + if responsehighlighter.HasBinaryContent(tracedata) { + hexDump = true + tracedata = hex.Dump([]byte(tracedata)) + } + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, tracedata, cliOptions.NoColor, hexDump) + gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) + } +} + // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { if _, err := url.ParseRequestURI(toTest); err != nil { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 3fbe3a8ec..17a87588d 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -252,7 +252,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } for _, input := range inputs { - if expressions.ContainsVariablesWithNames(input, map[string]interface{}{name: payload}) == nil { + if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input) == nil { hasPayloadName = true break } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 98037eed3..09dcfb451 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -170,7 +170,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { if input.Type.String() != "" { continue } - if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { + if expressions.ContainsVariablesWithNames(map[string]interface{}{name: payload}, input.Data) == nil { hasPayloadName = true break } From ccb588f383e200bff5bc1a3770efc393dadd59dc Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 13:53:42 +0000 Subject: [PATCH 130/196] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 13:53:42 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 37 +++++++++++++++++++++++++++++++ nuclei-jsonschema.json | 10 +++++++++ v2/pkg/templates/templates_doc.go | 28 ++++++++++++++++------- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index ed19bd9c0..7c26f3502 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2125,6 +2125,43 @@ retries: 5 ``` +
+ +
+ +
+ +trace bool + +
+
+ +Trace performs a trace operation for the target. + +
+ +
+ +
+ +trace-max-recursion int + +
+
+ +TraceMaxRecursion is the number of max recursion allowed for trace operations + + + +Examples: + + +```yaml +# Use a retry of 100 to 150 generally +trace-max-recursion: 100 +``` + +
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 9ace729ac..47714a35b 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -395,6 +395,16 @@ "title": "retries for dns request", "description": "Retries is the number of retries for the DNS request" }, + "trace": { + "type": "boolean", + "title": "trace operation", + "description": "Trace performs a trace operation for the target." + }, + "trace-max-recursion": { + "type": "integer", + "title": "trace-max-recursion level for dns request", + "description": "TraceMaxRecursion is the number of max recursion allowed for trace operations" + }, "recursion": { "type": "boolean", "title": "recurse all servers", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 52c5da75a..f64102821 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -757,7 +757,7 @@ func init() { FieldName: "dns", }, } - DNSRequestDoc.Fields = make([]encoder.Doc, 10) + DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" DNSRequestDoc.Fields[0].Note = "" @@ -825,16 +825,28 @@ func init() { DNSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Retries is the number of retries for the DNS request" DNSRequestDoc.Fields[7].AddExample("Use a retry of 3 to 5 generally", 5) - DNSRequestDoc.Fields[8].Name = "recursion" + DNSRequestDoc.Fields[8].Name = "trace" DNSRequestDoc.Fields[8].Type = "bool" DNSRequestDoc.Fields[8].Note = "" - DNSRequestDoc.Fields[8].Description = "Recursion determines if resolver should recurse all records to get fresh results." - DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." - DNSRequestDoc.Fields[9].Name = "resolvers" - DNSRequestDoc.Fields[9].Type = "[]string" + DNSRequestDoc.Fields[8].Description = "Trace performs a trace operation for the target." + DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Trace performs a trace operation for the target." + DNSRequestDoc.Fields[9].Name = "trace-max-recursion" + DNSRequestDoc.Fields[9].Type = "int" DNSRequestDoc.Fields[9].Note = "" - DNSRequestDoc.Fields[9].Description = "Resolvers to use for the dns requests" - DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" + DNSRequestDoc.Fields[9].Description = "TraceMaxRecursion is the number of max recursion allowed for trace operations" + DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = "TraceMaxRecursion is the number of max recursion allowed for trace operations" + + DNSRequestDoc.Fields[9].AddExample("Use a retry of 100 to 150 generally", 100) + DNSRequestDoc.Fields[10].Name = "recursion" + DNSRequestDoc.Fields[10].Type = "bool" + DNSRequestDoc.Fields[10].Note = "" + DNSRequestDoc.Fields[10].Description = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[10].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[11].Name = "resolvers" + DNSRequestDoc.Fields[11].Type = "[]string" + DNSRequestDoc.Fields[11].Note = "" + DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" + DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." From 2856e7e247d9a2ff3a6b067091847963f51f8eaa Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 18 Nov 2021 14:11:10 -0600 Subject: [PATCH 131/196] Extractor + Headless Actions structures to enums (#1217) * enum support for extractor --- v2/pkg/operators/extractors/compile.go | 10 +- .../operators/extractors/extractor_types.go | 105 ++++++++++ v2/pkg/operators/extractors/extractors.go | 29 +-- v2/pkg/protocols/dns/operators_test.go | 6 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 6 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/headless/engine/action.go | 108 +--------- .../protocols/headless/engine/action_types.go | 185 ++++++++++++++++++ .../protocols/headless/engine/page_actions.go | 4 +- .../headless/engine/page_actions_test.go | 119 +++++------ v2/pkg/protocols/http/operators_test.go | 14 +- v2/pkg/protocols/network/operators_test.go | 6 +- v2/pkg/protocols/network/request_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 6 +- v2/pkg/templates/templates_doc_examples.go | 4 +- 16 files changed, 382 insertions(+), 226 deletions(-) create mode 100644 v2/pkg/operators/extractors/extractor_types.go create mode 100644 v2/pkg/protocols/headless/engine/action_types.go diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 567716675..1d03184be 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -10,13 +10,12 @@ import ( // CompileExtractors performs the initial setup operation on an extractor func (e *Extractor) CompileExtractors() error { - var ok bool // Set up the extractor type - e.extractorType, ok = ExtractorTypes[e.Type] - if !ok { + computedType, err := toExtractorTypes(e.GetType().String()) + if err != nil { return fmt.Errorf("unknown extractor type specified: %s", e.Type) } - + e.extractorType = computedType // Compile the regexes for _, regex := range e.Regex { compiled, err := regexp.Compile(regex) @@ -25,7 +24,6 @@ func (e *Extractor) CompileExtractors() error { } e.regexCompiled = append(e.regexCompiled, compiled) } - for i, kval := range e.KVal { e.KVal[i] = strings.ToLower(kval) } @@ -43,7 +41,7 @@ func (e *Extractor) CompileExtractors() error { } if e.CaseInsensitive { - if e.Type != "kval" { + if e.GetType() != KValExtractor { return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type) } for i := range e.KVal { diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go new file mode 100644 index 000000000..227001a1f --- /dev/null +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -0,0 +1,105 @@ +package extractors + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// ExtractorType is the type of the extractor specified +type ExtractorType int + +const ( + // RegexExtractor extracts responses with regexes + RegexExtractor ExtractorType = iota + 1 + // KValExtractor extracts responses with key:value + KValExtractor + // XPathExtractor extracts responses with Xpath selectors + XPathExtractor + // JSONExtractor extracts responses with json + JSONExtractor + //limit + limit +) + +// extractorMappings is a table for conversion of extractor type from string. +var extractorMappings = map[ExtractorType]string{ + RegexExtractor: "regex", + KValExtractor: "kval", + XPathExtractor: "xpath", + JSONExtractor: "json", +} + +// GetType returns the type of the matcher +func (e *Extractor) GetType() ExtractorType { + return e.Type.ExtractorType +} + +// GetSupportedExtractorTypes returns list of supported types +func GetSupportedExtractorTypes() []ExtractorType { + var result []ExtractorType + for index := ExtractorType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toExtractorTypes(valueToMap string) (ExtractorType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range extractorMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid extractor type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ExtractorType) String() string { + return extractorMappings[t] +} + +// TypeHolder is used to hold internal type of the extractor +type TypeHolder struct { + ExtractorType ExtractorType +} + +func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the extractor", + Description: "Type of the extractor", + } + for _, types := range GetSupportedExtractorTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toExtractorTypes(marshalledTypes) + if err != nil { + return err + } + + holder.ExtractorType = computedType + return nil +} + +func (holder *TypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ExtractorType.String()) +} + +func (holder TypeHolder) MarshalYAML() (interface{}, error) { + return holder.ExtractorType.String(), nil +} diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index d4b2bb3c4..a2121efa5 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -21,7 +21,7 @@ type Extractor struct { // - "kval" // - "json" // - "xpath" - Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"` + Type TypeHolder `json:"name,omitempty" yaml:"type"` // extractorType is the internal type of the extractor extractorType ExtractorType @@ -113,30 +113,3 @@ type Extractor struct { // - true CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"` } - -// ExtractorType is the type of the extractor specified -type ExtractorType = int - -const ( - // RegexExtractor extracts responses with regexes - RegexExtractor ExtractorType = iota + 1 - // KValExtractor extracts responses with key:value - KValExtractor - // XPathExtractor extracts responses with Xpath selectors - XPathExtractor - // JSONExtractor extracts responses with json - JSONExtractor -) - -// ExtractorTypes is a table for conversion of extractor type from string. -var ExtractorTypes = map[string]ExtractorType{ - "regex": RegexExtractor, - "kval": KValExtractor, - "xpath": XPathExtractor, - "json": JSONExtractor, -} - -// GetType returns the type of the matcher -func (e *Extractor) GetType() ExtractorType { - return e.extractorType -} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 1937660c9..a34d6a622 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -192,7 +192,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -205,7 +205,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"rcode"}, } err = extractor.CompileExtractors() @@ -238,7 +238,7 @@ func TestDNSMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 38827f817..138fee6f2 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -35,7 +35,7 @@ func TestDNSExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index f85ff6c82..1f0b6da69 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -154,7 +154,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -167,7 +167,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"raw"}, } err = extractor.CompileExtractors() @@ -250,7 +250,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index bccfae728..fa0c8120c 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -37,7 +37,7 @@ func TestFileExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 5f97217b2..861f67a5e 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -2,110 +2,6 @@ package engine import "strings" -// ActionType defines the action type for a browser action -type ActionType int8 - -// Types to be executed by the user. -const ( - // ActionNavigate performs a navigation to the specified URL - // URL can include nuclei payload data such as URL, Hostname, etc. - ActionNavigate ActionType = iota + 1 - // ActionScript executes a JS snippet on the page. - ActionScript - // ActionClick performs the left-click action on an Element. - ActionClick - // ActionRightClick performs the right-click action on an Element. - ActionRightClick - // ActionTextInput performs an action for a text input - ActionTextInput - // ActionScreenshot performs the screenshot action writing to a file. - ActionScreenshot - // ActionTimeInput performs an action on a time input. - ActionTimeInput - // ActionSelectInput performs an action on a select input. - ActionSelectInput - // ActionFilesInput performs an action on a file input. - ActionFilesInput - // ActionWaitLoad waits for the page to stop loading. - ActionWaitLoad - // ActionGetResource performs a get resource action on an element - ActionGetResource - // ActionExtract performs an extraction on an element - ActionExtract - // ActionSetMethod sets the request method - ActionSetMethod - // ActionAddHeader adds a header to the request - ActionAddHeader - // ActionSetHeader sets a header in the request - ActionSetHeader - // ActionDeleteHeader deletes a header from the request - ActionDeleteHeader - // ActionSetBody sets the value of the request body - ActionSetBody - // ActionWaitEvent waits for a specific event. - ActionWaitEvent - // ActionKeyboard performs a keyboard action event on a page. - ActionKeyboard - // ActionDebug debug slows down headless and adds a sleep to each page. - ActionDebug - // ActionSleep executes a sleep for a specified duration - ActionSleep - // ActionWaitVisible waits until an element appears. - ActionWaitVisible -) - -// ActionStringToAction converts an action from string to internal representation -var ActionStringToAction = map[string]ActionType{ - "navigate": ActionNavigate, - "script": ActionScript, - "click": ActionClick, - "rightclick": ActionRightClick, - "text": ActionTextInput, - "screenshot": ActionScreenshot, - "time": ActionTimeInput, - "select": ActionSelectInput, - "files": ActionFilesInput, - "waitload": ActionWaitLoad, - "getresource": ActionGetResource, - "extract": ActionExtract, - "setmethod": ActionSetMethod, - "addheader": ActionAddHeader, - "setheader": ActionSetHeader, - "deleteheader": ActionDeleteHeader, - "setbody": ActionSetBody, - "waitevent": ActionWaitEvent, - "keyboard": ActionKeyboard, - "debug": ActionDebug, - "sleep": ActionSleep, - "waitvisible": ActionWaitVisible, -} - -// ActionToActionString converts an action from internal representation to string -var ActionToActionString = map[ActionType]string{ - ActionNavigate: "navigate", - ActionScript: "script", - ActionClick: "click", - ActionRightClick: "rightclick", - ActionTextInput: "text", - ActionScreenshot: "screenshot", - ActionTimeInput: "time", - ActionSelectInput: "select", - ActionFilesInput: "files", - ActionWaitLoad: "waitload", - ActionGetResource: "getresource", - ActionExtract: "extract", - ActionSetMethod: "set-method", - ActionAddHeader: "addheader", - ActionSetHeader: "setheader", - ActionDeleteHeader: "deleteheader", - ActionSetBody: "setbody", - ActionWaitEvent: "waitevent", - ActionKeyboard: "keyboard", - ActionDebug: "debug", - ActionSleep: "sleep", - ActionWaitVisible: "waitvisible", -} - // Action is an action taken by the browser to reach a navigation // // Each step that the browser executes is an action. Most navigations @@ -152,13 +48,13 @@ type Action struct { // - "keyboard" // - "debug" // - "sleep" - ActionType string `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` + ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` } // String returns the string representation of an action func (a *Action) String() string { builder := &strings.Builder{} - builder.WriteString(a.ActionType) + builder.WriteString(a.ActionType.String()) if a.Name != "" { builder.WriteString(" Name:") builder.WriteString(a.Name) diff --git a/v2/pkg/protocols/headless/engine/action_types.go b/v2/pkg/protocols/headless/engine/action_types.go new file mode 100644 index 000000000..4f71b4215 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/action_types.go @@ -0,0 +1,185 @@ +package engine + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// ActionType defines the action type for a browser action +type ActionType int8 + +// Types to be executed by the user. +const ( + // ActionNavigate performs a navigation to the specified URL + // URL can include nuclei payload data such as URL, Hostname, etc. + ActionNavigate ActionType = iota + 1 + // ActionScript executes a JS snippet on the page. + ActionScript + // ActionClick performs the left-click action on an Element. + ActionClick + // ActionRightClick performs the right-click action on an Element. + ActionRightClick + // ActionTextInput performs an action for a text input + ActionTextInput + // ActionScreenshot performs the screenshot action writing to a file. + ActionScreenshot + // ActionTimeInput performs an action on a time input. + ActionTimeInput + // ActionSelectInput performs an action on a select input. + ActionSelectInput + // ActionFilesInput performs an action on a file input. + ActionFilesInput + // ActionWaitLoad waits for the page to stop loading. + ActionWaitLoad + // ActionGetResource performs a get resource action on an element + ActionGetResource + // ActionExtract performs an extraction on an element + ActionExtract + // ActionSetMethod sets the request method + ActionSetMethod + // ActionAddHeader adds a header to the request + ActionAddHeader + // ActionSetHeader sets a header in the request + ActionSetHeader + // ActionDeleteHeader deletes a header from the request + ActionDeleteHeader + // ActionSetBody sets the value of the request body + ActionSetBody + // ActionWaitEvent waits for a specific event. + ActionWaitEvent + // ActionKeyboard performs a keyboard action event on a page. + ActionKeyboard + // ActionDebug debug slows down headless and adds a sleep to each page. + ActionDebug + // ActionSleep executes a sleep for a specified duration + ActionSleep + // ActionWaitVisible waits until an element appears. + ActionWaitVisible + // limit + limit +) + +// ActionStringToAction converts an action from string to internal representation +var ActionStringToAction = map[string]ActionType{ + "navigate": ActionNavigate, + "script": ActionScript, + "click": ActionClick, + "rightclick": ActionRightClick, + "text": ActionTextInput, + "screenshot": ActionScreenshot, + "time": ActionTimeInput, + "select": ActionSelectInput, + "files": ActionFilesInput, + "waitload": ActionWaitLoad, + "getresource": ActionGetResource, + "extract": ActionExtract, + "setmethod": ActionSetMethod, + "addheader": ActionAddHeader, + "setheader": ActionSetHeader, + "deleteheader": ActionDeleteHeader, + "setbody": ActionSetBody, + "waitevent": ActionWaitEvent, + "keyboard": ActionKeyboard, + "debug": ActionDebug, + "sleep": ActionSleep, + "waitvisible": ActionWaitVisible, +} + +// ActionToActionString converts an action from internal representation to string +var ActionToActionString = map[ActionType]string{ + ActionNavigate: "navigate", + ActionScript: "script", + ActionClick: "click", + ActionRightClick: "rightclick", + ActionTextInput: "text", + ActionScreenshot: "screenshot", + ActionTimeInput: "time", + ActionSelectInput: "select", + ActionFilesInput: "files", + ActionWaitLoad: "waitload", + ActionGetResource: "getresource", + ActionExtract: "extract", + ActionSetMethod: "set-method", + ActionAddHeader: "addheader", + ActionSetHeader: "setheader", + ActionDeleteHeader: "deleteheader", + ActionSetBody: "setbody", + ActionWaitEvent: "waitevent", + ActionKeyboard: "keyboard", + ActionDebug: "debug", + ActionSleep: "sleep", + ActionWaitVisible: "waitvisible", +} + +// GetSupportedActionTypes returns list of supported types +func GetSupportedActionTypes() []ActionType { + var result []ActionType + for index := ActionType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toActionTypes(valueToMap string) (ActionType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range ActionToActionString { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid action type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ActionType) String() string { + return ActionToActionString[t] +} + +// ActionTypeHolder is used to hold internal type of the action +type ActionTypeHolder struct { + ActionType ActionType +} + +func (holder ActionTypeHolder) String() string { + return holder.ActionType.String() +} +func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "action to perform", + Description: "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep", + } + for _, types := range GetSupportedActionTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *ActionTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toActionTypes(marshalledTypes) + if err != nil { + return err + } + + holder.ActionType = computedType + return nil +} + +func (holder *ActionTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ActionType.String()) +} + +func (holder ActionTypeHolder) MarshalYAML() (interface{}, error) { + return holder.ActionType.String(), nil +} diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index fc6e821ea..234d726eb 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -24,9 +24,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s outData := make(map[string]string) for _, act := range actions { - actionType := ActionStringToAction[act.ActionType] - - switch actionType { + switch act.ActionType.ActionType { case ActionNavigate: err = p.NavigateURL(act, outData, baseURL) case ActionScript: diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 902b978ac..f9819756d 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) { ` - actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} + actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -50,10 +50,11 @@ func TestActionScript(t *testing.T) { t.Run("run-and-results", func(t *testing.T) { actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") @@ -63,10 +64,10 @@ func TestActionScript(t *testing.T) { t.Run("hook", func(t *testing.T) { actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -87,9 +88,9 @@ func TestActionClick(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -120,9 +121,9 @@ func TestActionRightClick(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -145,9 +146,9 @@ func TestActionTextInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -162,9 +163,9 @@ func TestActionTextInput(t *testing.T) { func TestActionHeadersChange(t *testing.T) { actions := []*Action{ - {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -189,9 +190,9 @@ func TestActionScreenshot(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -214,9 +215,9 @@ func TestActionTimeInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -242,9 +243,9 @@ func TestActionSelectInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -265,9 +266,9 @@ func TestActionFilesInput(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -291,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -316,8 +317,8 @@ func TestActionGetResource(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -336,8 +337,8 @@ func TestActionExtract(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -355,8 +356,8 @@ func TestActionSetMethod(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -367,9 +368,9 @@ func TestActionSetMethod(t *testing.T) { func TestActionAddHeader(t *testing.T) { actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -386,11 +387,11 @@ func TestActionAddHeader(t *testing.T) { func TestActionDeleteHeader(t *testing.T) { actions := []*Action{ - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, - {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, - {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -407,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) { func TestActionSetBody(t *testing.T) { actions := []*Action{ - {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -435,10 +436,10 @@ func TestActionKeyboard(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "input"}}, - {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -461,8 +462,8 @@ func TestActionSleep(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -484,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) { ` actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, } t.Run("wait for an element being visible", func(t *testing.T) { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 7950bc64f..50e9c9d04 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -166,7 +166,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -179,7 +179,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test_header"}, } err = extractor.CompileExtractors() @@ -195,7 +195,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("jq-simple", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters | .batter | .[] | .id"}, } err = extractor.CompileExtractors() @@ -207,7 +207,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-array", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".array"}, } err = extractor.CompileExtractors() @@ -219,7 +219,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-object", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "json", + Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters"}, } err = extractor.CompileExtractors() @@ -235,7 +235,7 @@ func TestHTTPOperatorExtract(t *testing.T) { event["body"] = exampleResponseBody extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"TEST_HEADER"}, // only applies to KVal CaseInsensitive: true, } @@ -267,7 +267,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 9e8a193c3..292473bf5 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -149,7 +149,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -162,7 +162,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"request"}, } err = extractor.CompileExtractors() @@ -193,7 +193,7 @@ func TestNetworkMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 0e8d194bd..1e51b9c6f 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -38,7 +38,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"

.*

"}, }}, }, diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 05fe9468f..1290dfb7f 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -139,7 +139,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -152,7 +152,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: "kval", + Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test-header"}, Part: "header", } @@ -184,7 +184,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: "regex", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }} diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 88d5e5e88..c29766576 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -44,7 +44,7 @@ var ( Recursion: true, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, + {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, }, }, } @@ -54,7 +54,7 @@ var ( Extensions: []string{"all"}, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: "regex", Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, + {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, }, }, } From 5a0f483cd3d59f86fef956b815ba4dfffaa4b676 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 18 Nov 2021 20:12:46 +0000 Subject: [PATCH 132/196] Auto Generate Syntax Docs + JSONSchema [Thu Nov 18 20:12:46 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 4 +- nuclei-jsonschema.json | 77 ++++++++++++++++++------------- v2/pkg/templates/templates_doc.go | 4 +- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 7c26f3502..6ceae4337 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1658,7 +1658,7 @@ name: cookie-extractor
-type string +type TypeHolder
@@ -2879,7 +2879,7 @@ Description is the optional description of the headless action
-action string +action ActionTypeHolder
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 47714a35b..ddaf889fc 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -130,15 +130,8 @@ "description": "Name of the extractor" }, "type": { - "enum": [ - "regex", - "kval", - "json", - "xpath" - ], - "type": "string", - "title": "type of the extractor", - "description": "Type of the extractor" + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/extractors.TypeHolder" }, "regex": { "items": { @@ -201,6 +194,17 @@ "additionalProperties": false, "type": "object" }, + "extractors.TypeHolder": { + "enum": [ + "regex", + "kval", + "xpath", + "json" + ], + "type": "string", + "title": "type of the extractor", + "description": "Type of the extractor" + }, "matchers.Matcher": { "required": [ "type" @@ -555,30 +559,8 @@ "description": "Description of the headless action" }, "action": { - "enum": [ - "navigate", - "script", - "click", - "rightclick", - "text", - "screenshot", - "time", - "select", - "files", - "waitload", - "getresource", - "extract", - "setmethod", - "addheader", - "setheader", - "deleteheader", - "setbody", - "waitevent", - "keyboard", - "debug", - "sleep" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/engine.ActionTypeHolder", "title": "action to perform", "description": "Type of actions to perform" } @@ -586,6 +568,35 @@ "additionalProperties": false, "type": "object" }, + "engine.ActionTypeHolder": { + "enum": [ + "navigate", + "script", + "click", + "rightclick", + "text", + "screenshot", + "time", + "select", + "files", + "waitload", + "getresource", + "extract", + "set-method", + "addheader", + "setheader", + "deleteheader", + "setbody", + "waitevent", + "keyboard", + "debug", + "sleep", + "waitvisible" + ], + "type": "string", + "title": "action to perform", + "description": "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep" + }, "http.HTTPMethodTypeHolder": { "enum": [ "GET", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index f64102821..df4ff0a40 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -649,7 +649,7 @@ func init() { EXTRACTORSExtractorDoc.Fields[0].AddExample("", "cookie-extractor") EXTRACTORSExtractorDoc.Fields[1].Name = "type" - EXTRACTORSExtractorDoc.Fields[1].Type = "string" + EXTRACTORSExtractorDoc.Fields[1].Type = "TypeHolder" EXTRACTORSExtractorDoc.Fields[1].Note = "" EXTRACTORSExtractorDoc.Fields[1].Description = "Type is the type of the extractor." EXTRACTORSExtractorDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of the extractor." @@ -1098,7 +1098,7 @@ func init() { ENGINEActionDoc.Fields[2].Description = "Description is the optional description of the headless action" ENGINEActionDoc.Fields[2].Comments[encoder.LineComment] = "Description is the optional description of the headless action" ENGINEActionDoc.Fields[3].Name = "action" - ENGINEActionDoc.Fields[3].Type = "string" + ENGINEActionDoc.Fields[3].Type = "ActionTypeHolder" ENGINEActionDoc.Fields[3].Note = "" ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." From 883cb22c39f0eb70a8aba2ad27f59c48912a6bcf Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 19 Nov 2021 14:46:10 +0530 Subject: [PATCH 133/196] Adding more resources --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e4626a05a..efe80f381 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,8 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu ### Resources + +- [Scanning Live Web Applications with Nuclei in CI/CD Pipeline](https://blog.escape.tech/devsecops-part-iii-scanning-live-web-applications/) by [@TristanKalos](https://twitter.com/TristanKalos) - [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/) - [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/) - [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/) From 09eba6c0cf7b27c8210eb327f0b2c7a6e5e8142c Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 19 Nov 2021 12:15:43 +0200 Subject: [PATCH 134/196] feat: In case of binary data, show a hexadecimal view as well #1080 (#1266) Highlight the longest occurrences when there are matches that are substrings of each other --- .../response_highlighter.go | 40 +++++++--- .../response_highlighter_test.go | 77 +++++++++++++------ 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go index f637644f0..ccec3cd38 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -1,6 +1,7 @@ package responsehighlighter import ( + "sort" "strconv" "strings" @@ -14,18 +15,14 @@ var colorFunction = aurora.Green func Highlight(operatorResult *operators.Result, response string, noColor, hexDump bool) string { result := response if operatorResult != nil && !noColor { - for _, matches := range operatorResult.Matches { - if len(matches) > 0 { - for _, currentMatch := range matches { - if hexDump { - highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) - if err == nil { - result = highlightedHexDump.String() - } - } else { - result = strings.ReplaceAll(result, currentMatch, addColor(currentMatch)) - } + for _, currentMatch := range getSortedMatches(operatorResult) { + if hexDump { + highlightedHexDump, err := toHighLightedHexDump(result, currentMatch) + if err == nil { + result = highlightedHexDump.String() } + } else { + result = highlightASCII(currentMatch, result) } } } @@ -33,6 +30,27 @@ func Highlight(operatorResult *operators.Result, response string, noColor, hexDu return result } +func highlightASCII(currentMatch string, result string) string { + var coloredMatchBuilder strings.Builder + for _, char := range currentMatch { + coloredMatchBuilder.WriteString(addColor(string(char))) + } + + return strings.ReplaceAll(result, currentMatch, coloredMatchBuilder.String()) +} + +func getSortedMatches(operatorResult *operators.Result) []string { + sortedMatches := make([]string, 0, len(operatorResult.Matches)) + for _, matches := range operatorResult.Matches { + sortedMatches = append(sortedMatches, matches...) + } + + sort.Slice(sortedMatches, func(i, j int) bool { + return len(sortedMatches[i]) > len(sortedMatches[j]) + }) + return sortedMatches +} + func CreateStatusCodeSnippet(response string, statusCode int) string { if strings.HasPrefix(response, "HTTP/") { strStatusCode := strconv.Itoa(statusCode) diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go index a60341ef1..fcdd16a84 100644 --- a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter_test.go @@ -12,15 +12,16 @@ import ( const input = "abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn" func TestHexDumpHighlighting(t *testing.T) { - const highlightedHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| -00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| -00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| -00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| -00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| -00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| -00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| -00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| -` + highlightedHexDumpResponse := + "00000000 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 |abc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnab|\n" + + "00000010 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m |c\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m|\n" + + "00000020 \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m |\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m|\n" + + "00000030 \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m |\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m|\n" + + "00000040 \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m |\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0m|\n" + + "00000050 6b 6c 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c |klmnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mkl|\n" + + "00000060 6d 6e 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |mnabc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + + "00000070 61 62 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |abc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + t.Run("Test highlighting when the snippet is wrapped", func(t *testing.T) { result, err := toHighLightedHexDump(hex.Dump([]byte(input)), "defghij") assert.Nil(t, err) @@ -31,24 +32,25 @@ func TestHexDumpHighlighting(t *testing.T) { value := "asdfasdfasda|basdfadsdfs|" result, err := toHighLightedHexDump(hex.Dump([]byte(value)), "a|b") - expected := `00000000 61 73 64 66 61 73 64 66 61 73 64 61 7c 62 61 73 |asdfasdfasda|bas| -00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs|| -` + expected := + "00000000 61 73 64 66 61 73 64 66 61 73 64 \x1b[32m61\x1b[0m \x1b[32m7c\x1b[0m \x1b[32m62\x1b[0m 61 73 |asdfasdfasd\x1b[32ma\x1b[0m\x1b[32m|\x1b[0m\x1b[32mb\x1b[0mas|\n" + + "00000010 64 66 61 64 73 64 66 73 7c |dfadsdfs||\n" + assert.Nil(t, err) assert.Equal(t, expected, result.String()) }) } func TestHighlight(t *testing.T) { - const multiSnippetHighlightHexDumpResponse = `00000000 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 |abcdefghijklmnab| -00000010 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 |cdefghijklmnabcd| -00000020 65 66 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 |efghijklmnabcdef| -00000030 67 68 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 |ghijklmnabcdefgh| -00000040 69 6a 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a |ijklmnabcdefghij| -00000050 6b 6c 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c |klmnabcdefghijkl| -00000060 6d 6e 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |mnabcdefghijklmn| -00000070 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e |abcdefghijklmn| -` + const multiSnippetHighlightHexDumpResponse = "00000000 \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m |\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0m|\n" + + "00000010 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m |c\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m|\n" + + "00000020 \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m |\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m|\n" + + "00000030 \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m |\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m|\n" + + "00000040 \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m |\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0m|\n" + + "00000050 6b 6c 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c |klmn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mkl|\n" + + "00000060 6d 6e \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |mn\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + + "00000070 \x1b[32m61\x1b[0m \x1b[32m62\x1b[0m 63 \x1b[32m64\x1b[0m \x1b[32m65\x1b[0m \x1b[32m66\x1b[0m \x1b[32m67\x1b[0m \x1b[32m68\x1b[0m \x1b[32m69\x1b[0m \x1b[32m6a\x1b[0m 6b 6c 6d 6e |\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn|\n" + matches := map[string][]string{ "first": {"defghij"}, "second": {"ab"}, @@ -62,7 +64,17 @@ func TestHighlight(t *testing.T) { t.Run("Test highlighting without hexdump", func(t *testing.T) { result := Highlight(&operatorResult, input, false, false) - expected := `abcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmnabcdefghijklmn` + expected := + "\x1b[32ma\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn\x1b[32m" + + "a\x1b[0m\x1b[32mb\x1b[0mc\x1b[32md\x1b[0m\x1b[32me\x1b[0m\x1b[32mf\x1b[0m\x1b[32mg\x1b[0m\x1b[32mh\x1b[0m\x1b[32mi\x1b[0m\x1b[32mj\x1b[0mklmn" + print(result) assert.Equal(t, expected, result) }) @@ -76,3 +88,24 @@ func TestHighlight(t *testing.T) { assert.Equal(t, hex.Dump([]byte(input)), result) }) } + +func TestMultiSubstringMatchHighlight(t *testing.T) { + const input = ` +start ValueToMatch end +start ValueToMatch-1.2.3 end +start ValueToMatch-2.1 end +` + matches := map[string][]string{ + "first": {"ValueToMatch"}, + "second": {"ValueToMatch-1.2.3"}, + "third": {"ValueToMatch-2.1"}, + } + operatorResult := operators.Result{Matches: matches} + + expected := + "\nstart \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m end\n" + + "start \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m\x1b[32m-\x1b[0m\x1b[32m1\x1b[0m\x1b[32m.\x1b[0m\x1b[32m2\x1b[0m\x1b[32m.\x1b[0m\x1b[32m3\x1b[0m end\n" + + "start \x1b[32mV\x1b[0m\x1b[32ma\x1b[0m\x1b[32ml\x1b[0m\x1b[32mu\x1b[0m\x1b[32me\x1b[0m\x1b[32mT\x1b[0m\x1b[32mo\x1b[0m\x1b[32mM\x1b[0m\x1b[32ma\x1b[0m\x1b[32mt\x1b[0m\x1b[32mc\x1b[0m\x1b[32mh\x1b[0m\x1b[32m-\x1b[0m\x1b[32m2\x1b[0m\x1b[32m.\x1b[0m\x1b[32m1\x1b[0m end \n" + result := Highlight(&operatorResult, input, false, false) + assert.Equal(t, expected, result) +} From 4b8ec29d8d788d45543570ca97bbc2e654a4779a Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Fri, 19 Nov 2021 04:54:09 -0600 Subject: [PATCH 135/196] struct to enums changes for Matchers (#1246) * struct to enums changes for Matchers --- v2/pkg/operators/matchers/compile.go | 13 +- v2/pkg/operators/matchers/match_test.go | 6 +- v2/pkg/operators/matchers/matchers.go | 33 +---- v2/pkg/operators/matchers/matchers_types.go | 115 ++++++++++++++++++ v2/pkg/operators/operators.go | 2 +- v2/pkg/protocols/dns/operators_test.go | 12 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 12 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/http/operators_test.go | 10 +- v2/pkg/protocols/network/operators_test.go | 10 +- v2/pkg/protocols/network/request_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 8 +- v2/pkg/templates/templates_doc_examples.go | 8 +- 14 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 v2/pkg/operators/matchers/matchers_types.go diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 0efebb1aa..79d062a7c 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -25,11 +25,18 @@ func (m *Matcher) CompileMatchers() error { } // Set up the matcher type - m.matcherType, ok = MatcherTypes[m.Type] - if !ok { + computedType, err := toMatcherTypes(m.GetType().String()) + if err != nil { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } + m.matcherType = computedType + // By default, match on body if user hasn't provided any specific items + if m.Part == "" { + m.Part = "body" + } + + // Compile the regexes for _, regex := range m.Regex { compiled, err := regexp.Compile(regex) @@ -68,7 +75,7 @@ func (m *Matcher) CompileMatchers() error { } if m.CaseInsensitive { - if m.Type != "word" { + if m.GetType() != WordsMatcher { return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) } for i := range m.Words { diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 4c2f9bc0c..b52adbc6d 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -19,7 +19,7 @@ func TestWordANDCondition(t *testing.T) { } func TestRegexANDCondition(t *testing.T) { - m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} + m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} err := m.CompileMatchers() require.Nil(t, err) @@ -49,7 +49,7 @@ func TestORCondition(t *testing.T) { } func TestRegexOrCondition(t *testing.T) { - m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} + m := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} err := m.CompileMatchers() require.Nil(t, err) @@ -63,7 +63,7 @@ func TestRegexOrCondition(t *testing.T) { } func TestHexEncoding(t *testing.T) { - m := &Matcher{Encoding: "hex", Type: "word", Part: "body", Words: []string{"50494e47"}} + m := &Matcher{Encoding: "hex", Type: MatcherTypeHolder{MatcherType: WordsMatcher}, Part: "body", Words: []string{"50494e47"}} err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 4c0fd1fee..3f37deb02 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -17,7 +17,7 @@ type Matcher struct { // - "regex" // - "binary" // - "dsl" - Type string `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` + Type MatcherTypeHolder `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` // description: | // Condition is the optional condition between two matcher variables. By default, // the condition is assumed to be OR. @@ -120,33 +120,6 @@ type Matcher struct { dslCompiled []*govaluate.EvaluableExpression } -// MatcherType is the type of the matcher specified -type MatcherType = int - -const ( - // WordsMatcher matches responses with words - WordsMatcher MatcherType = iota + 1 - // RegexMatcher matches responses with regexes - RegexMatcher - // BinaryMatcher matches responses with words - BinaryMatcher - // StatusMatcher matches responses with status codes - StatusMatcher - // SizeMatcher matches responses with response size - SizeMatcher - // DSLMatcher matches based upon dsl syntax - DSLMatcher -) - -// MatcherTypes is a table for conversion of matcher type from string. -var MatcherTypes = map[string]MatcherType{ - "status": StatusMatcher, - "size": SizeMatcher, - "word": WordsMatcher, - "regex": RegexMatcher, - "binary": BinaryMatcher, - "dsl": DSLMatcher, -} // ConditionType is the type of condition for matcher type ConditionType int @@ -180,7 +153,3 @@ func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) ( return data, matchedSnippet } -// GetType returns the type of the matcher -func (m *Matcher) GetType() MatcherType { - return m.matcherType -} diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go new file mode 100644 index 000000000..312d4f59f --- /dev/null +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -0,0 +1,115 @@ +package matchers + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/alecthomas/jsonschema" +) + +// MatcherType is the type of the matcher specified +type MatcherType int + +const ( + // WordsMatcher matches responses with words + WordsMatcher MatcherType = iota + 1 + // RegexMatcher matches responses with regexes + RegexMatcher + // BinaryMatcher matches responses with words + BinaryMatcher + // StatusMatcher matches responses with status codes + StatusMatcher + // SizeMatcher matches responses with response size + SizeMatcher + // DSLMatcher matches based upon dsl syntax + DSLMatcher + //limit + limit +) + +// MatcherTypes is a table for conversion of matcher type from string. +var MatcherTypes = map[MatcherType]string{ + StatusMatcher: "status", + SizeMatcher: "size", + WordsMatcher: "word", + RegexMatcher: "regex", + BinaryMatcher: "binary", + DSLMatcher: "dsl", +} + +//GetType returns the type of the matcher +func (e *Matcher) GetType() MatcherType { + return e.Type.MatcherType +} + +// GetSupportedMatcherTypes returns list of supported types +func GetSupportedMatcherTypes() []MatcherType { + var result []MatcherType + for index := MatcherType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toMatcherTypes(valueToMap string) (MatcherType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range MatcherTypes { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid matcher type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t MatcherType) String() string { + return MatcherTypes[t] +} + +// MatcherTypeHolder is used to hold internal type of the matcher +type MatcherTypeHolder struct { + MatcherType MatcherType +} + +func (t MatcherTypeHolder) String() string { + return t.MatcherType.String() +} + +func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the matcher", + Description: "Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl", + } + for _, types := range GetSupportedMatcherTypes() { + gotType.Enum = append(gotType.Enum, types) + } + return gotType +} + +func (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toMatcherTypes(marshalledTypes) + if err != nil { + return err + } + + holder.MatcherType = computedType + return nil +} + +func (holder *MatcherTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MatcherType) +} + +func (holder MatcherTypeHolder) MarshalYAML() (interface{}, error) { + return holder.MatcherType, nil +} diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index af13b4d93..0b9087312 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -179,7 +179,7 @@ func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { if matcher.Name != "" { return matcher.Name } else { - return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable + return matcher.Type.String() + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable } } diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index a34d6a622..aef43ccfa 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -81,7 +81,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -95,7 +95,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("rcode", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "rcode", - Type: "status", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher}, Status: []int{dns.RcodeSuccess}, } err = matcher.CompileMatchers() @@ -109,7 +109,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -124,7 +124,7 @@ func TestDNSOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -147,7 +147,7 @@ func TestDNSOperatorMatch(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"one.ONE.one.ONE"}, CaseInsensitive: true, } @@ -233,7 +233,7 @@ func TestDNSMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 138fee6f2..7b6a3e6ab 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -30,7 +30,7 @@ func TestDNSExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"93.184.216.34"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 1f0b6da69..ac41a9400 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -66,7 +66,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -80,7 +80,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -95,7 +95,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -114,7 +114,7 @@ func TestFileOperatorMatch(t *testing.T) { matcher := &matchers.Matcher{ Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"TeSt-DaTA"}, CaseInsensitive: true, } @@ -200,13 +200,13 @@ func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output. matcher := []*matchers.Matcher{ { Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: expectedValue, }, { Name: namedMatcherName, Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: expectedValue, }, } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index fa0c8120c..ba057dafe 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -32,7 +32,7 @@ func TestFileExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "raw", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 50e9c9d04..7aefd7d19 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -78,7 +78,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -92,7 +92,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -107,7 +107,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -121,7 +121,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", // only applies to word + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, // only applies to word Words: []string{"EXAMPLE DOMAIN"}, CaseInsensitive: true, } @@ -262,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 292473bf5..daa92725e 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -64,7 +64,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"STAT "}, } err = matcher.CompileMatchers() @@ -78,7 +78,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -93,7 +93,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -107,7 +107,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"rESp-DAta"}, CaseInsensitive: true, } @@ -188,7 +188,7 @@ func TestNetworkMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"STAT "}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 1e51b9c6f..3cbff4d96 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -33,7 +33,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "data", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"200 OK"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 1290dfb7f..b69ccbdc2 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -70,7 +70,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("valid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, } err = matcher.CompileMatchers() @@ -84,7 +84,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Negative: true, Words: []string{"random"}, } @@ -99,7 +99,7 @@ func TestHTTPOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"random"}, } err := matcher.CompileMatchers() @@ -179,7 +179,7 @@ func TestHTTPMakeResult(t *testing.T) { Matchers: []*matchers.Matcher{{ Name: "test", Part: "body", - Type: "word", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, Words: []string{"1.1.1.1"}, }}, Extractors: []*extractors.Extractor{{ diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index c29766576..49e58eb00 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -29,9 +29,9 @@ var ( Operators: operators.Operators{ MatchersCondition: "and", Matchers: []*matchers.Matcher{ - {Type: "word", Words: []string{"[core]"}}, - {Type: "dsl", DSL: []string{"!contains(tolower(body), ' Date: Fri, 19 Nov 2021 10:55:27 +0000 Subject: [PATCH 136/196] Auto Generate Syntax Docs + JSONSchema [Fri Nov 19 10:55:27 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 18 +++++++++--------- nuclei-jsonschema.json | 24 +++++++++++++++--------- v2/pkg/templates/templates_doc.go | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 6ceae4337..c3fb0b637 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: word + - type: 1 words: - '[core]' - - type: dsl + - type: 6 condition: and dsl: - '!contains(tolower(body), '' -type string +type MatcherTypeHolder
@@ -2396,7 +2396,7 @@ inputs: - data: "envi\r\nquit\r\n" read-size: 2048 matchers: - - type: word + - type: 1 words: - zookeeper.version ``` diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index ddaf889fc..32d88583d 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -211,15 +211,8 @@ ], "properties": { "type": { - "enum": [ - "status", - "size", - "word", - "regex", - "binary", - "dsl" - ], - "type": "string", + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/matchers.MatcherTypeHolder", "title": "type of matcher", "description": "Type of the matcher" }, @@ -312,6 +305,19 @@ "additionalProperties": false, "type": "object" }, + "matchers.MatcherTypeHolder": { + "enum": [ + 1, + 2, + 3, + 4, + 5, + 6 + ], + "type": "string", + "title": "type of the matcher", + "description": "Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl" + }, "generators.AttackTypeHolder": { "enum": [ "batteringram", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index df4ff0a40..eb547d0a6 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -497,7 +497,7 @@ func init() { } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" - MATCHERSMatcherDoc.Fields[0].Type = "string" + MATCHERSMatcherDoc.Fields[0].Type = "MatcherTypeHolder" MATCHERSMatcherDoc.Fields[0].Note = "" MATCHERSMatcherDoc.Fields[0].Description = "Type is the type of the matcher." MATCHERSMatcherDoc.Fields[0].Comments[encoder.LineComment] = "Type is the type of the matcher." From f74ff3fc496c7e5b6980cd9bf0fe740ef64b02e5 Mon Sep 17 00:00:00 2001 From: Sajad Date: Sat, 20 Nov 2021 13:25:27 +0530 Subject: [PATCH 137/196] Tag based struct validation (#1256) * Added tag based struct validation --- .../test-issue-tracker-config1.yaml | 4 +- .../test-issue-tracker-config2.yaml | 4 +- v2/go.mod | 5 +++ v2/go.sum | 14 +++++++ v2/internal/runner/options.go | 15 ++++++- v2/internal/runner/runner.go | 6 +-- .../reporting/exporters/es/elasticsearch.go | 40 +++---------------- v2/pkg/reporting/trackers/github/github.go | 36 +++-------------- v2/pkg/reporting/trackers/gitlab/gitlab.go | 34 ++-------------- v2/pkg/reporting/trackers/jira/jira.go | 40 +++---------------- v2/pkg/types/types.go | 2 +- v2/pkg/utils/yaml/yaml_decode_wrapper.go | 34 ++++++++++++++++ 12 files changed, 94 insertions(+), 140 deletions(-) create mode 100644 v2/pkg/utils/yaml/yaml_decode_wrapper.go diff --git a/integration_tests/test-issue-tracker-config1.yaml b/integration_tests/test-issue-tracker-config1.yaml index b7c1f73ce..dd8ceb180 100644 --- a/integration_tests/test-issue-tracker-config1.yaml +++ b/integration_tests/test-issue-tracker-config1.yaml @@ -26,8 +26,8 @@ gitlab: username: test-username # token is the token for gitlab account. token: test-token - # project-id is the ID of the repository. - project-id: 1234 + # project-name is the name/id of the project(repository). + project-name: "1234" # issue-label is the label of the created issue type issue-label: bug diff --git a/integration_tests/test-issue-tracker-config2.yaml b/integration_tests/test-issue-tracker-config2.yaml index eeb6eaa37..c76b773eb 100644 --- a/integration_tests/test-issue-tracker-config2.yaml +++ b/integration_tests/test-issue-tracker-config2.yaml @@ -28,8 +28,8 @@ gitlab: username: test-username # token is the token for gitlab account. token: test-token - # project-id is the ID of the repository. - project-id: 1234 + # project-name is the name/id of the project(repository). + project-name: "1234" # issue-label is the label of the created issue type issue-label: bug diff --git a/v2/go.mod b/v2/go.mod index c7d51d01e..74eb7ea3f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,6 +12,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.1.1 + github.com/go-playground/validator/v10 v10.9.0 github.com/go-rod/rod v0.101.8 github.com/gobwas/ws v1.1.0 github.com/google/go-github v17.0.0+incompatible @@ -82,6 +83,8 @@ require ( github.com/eggsampler/acme/v3 v3.2.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect @@ -100,6 +103,7 @@ require ( github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -119,6 +123,7 @@ require ( github.com/ysmood/goob v0.3.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index c649ceb45..f474a7273 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -230,6 +230,14 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= @@ -456,6 +464,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -841,6 +851,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1014,8 +1026,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index b4dc2f3df..f11d2055c 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -2,11 +2,13 @@ package runner import ( "bufio" - "errors" "os" "path/filepath" "strings" + "github.com/pkg/errors" + + "github.com/go-playground/validator/v10" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -77,6 +79,17 @@ func hasStdin() bool { // validateOptions validates the configuration options passed func validateOptions(options *types.Options) error { + validate := validator.New() + if err := validate.Struct(options); err != nil { + if _, ok := err.(*validator.InvalidValidationError); ok { + return err + } + errs := []string{} + for _, err := range err.(validator.ValidationErrors) { + errs = append(errs, err.Namespace()+": "+err.Tag()) + } + return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields") + } if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 5e71b76fe..9e9ee8f19 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -10,7 +10,6 @@ import ( "github.com/logrusorgru/aurora" "github.com/pkg/errors" "go.uber.org/ratelimit" - "gopkg.in/yaml.v2" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" @@ -35,6 +34,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" + yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml" ) // Runner is a client for running the enumeration process. @@ -180,9 +180,9 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } reportingOptions = &reporting.Options{} - if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil { + if err := yamlwrapper.DecodeAndValidate(file, reportingOptions); err != nil { file.Close() - return nil, errors.Wrap(parseErr, "could not parse reporting config file") + return nil, errors.Wrap(err, "could not parse reporting config file") } file.Close() } diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 784e587d0..0959cc929 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" "time" "encoding/base64" @@ -20,19 +19,19 @@ import ( // Options contains necessary options required for elasticsearch communicaiton type Options struct { // IP for elasticsearch instance - IP string `yaml:"ip"` + IP string `yaml:"ip" validate:"required,ip"` // Port is the port of elasticsearch instance - Port int `yaml:"port"` + Port int `yaml:"port" validate:"required,gte=0,lte=65535"` // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` // Username for the elasticsearch instance - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Password is the password for elasticsearch instance - Password string `yaml:"password"` + Password string `yaml:"password" validate:"required"` // IndexName is the name of the elasticsearch index - IndexName string `yaml:"index-name"` + IndexName string `yaml:"index-name" validate:"required"` } type data struct { @@ -50,10 +49,6 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter - err := validateOptions(option) - if err != nil { - return nil, err - } client := &http.Client{ Timeout: 5 * time.Second, @@ -86,31 +81,6 @@ func New(option *Options) (*Exporter, error) { return ei, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.IP == "" { - errs = append(errs, "IP") - } - if options.Port == 0 { - errs = append(errs, "Port") - } - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Password == "" { - errs = append(errs, "Password") - } - if options.IndexName == "" { - errs = append(errs, "IndexName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 8d6cdb864..47225ada1 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -24,15 +24,15 @@ type Integration struct { // Options contains the configuration options for github issue tracker client type Options struct { // BaseURL (optional) is the self-hosted github application url - BaseURL string `yaml:"base-url"` + BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the github user - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Owner (manadatory) is the owner name of the repository for issues. - Owner string `yaml:"owner"` + Owner string `yaml:"owner" validate:"required"` // Token is the token for github account. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -42,10 +42,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: options.Token}, @@ -66,28 +62,6 @@ func New(options *Options) (*Integration, error) { return &Integration{client: client, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Owner == "" { - errs = append(errs, "Owner") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index 35922970b..c1438e2dc 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,9 +2,6 @@ package gitlab import ( "fmt" - "strings" - - "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" @@ -21,13 +18,13 @@ type Integration struct { // Options contains the configuration options for gitlab issue tracker client type Options struct { // BaseURL (optional) is the self-hosted gitlab application url - BaseURL string `yaml:"base-url"` + BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the gitlab user - Username string `yaml:"username"` + Username string `yaml:"username" validate:"required"` // Token is the token for gitlab account. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueLabel is the label of the created issue type IssueLabel string `yaml:"issue-label"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -37,10 +34,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) @@ -56,25 +49,6 @@ func New(options *Options) (*Integration, error) { return &Integration{client: git, userID: user.ID, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.Username == "" { - errs = append(errs, "Username") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 3237e929b..16ef1b2a7 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -2,7 +2,6 @@ package jira import ( "bytes" - "errors" "fmt" "io/ioutil" "strings" @@ -28,15 +27,15 @@ type Options struct { // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` // URL is the URL of the jira server - URL string `yaml:"url"` + URL string `yaml:"url" validate:"required"` // AccountID is the accountID of the jira user. - AccountID string `yaml:"account-id"` + AccountID string `yaml:"account-id" validate:"required"` // Email is the email of the user for jira instance - Email string `yaml:"email"` + Email string `yaml:"email" validate:"required,email"` // Token is the token for jira instance. - Token string `yaml:"token"` + Token string `yaml:"token" validate:"required"` // ProjectName is the name of the project. - ProjectName string `yaml:"project-name"` + ProjectName string `yaml:"project-name" validate:"required"` // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` // SeverityAsLabel (optional) sends the severity as the label of the created @@ -46,10 +45,6 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { - err := validateOptions(options) - if err != nil { - return nil, err - } username := options.Email if !options.Cloud { username = options.AccountID @@ -65,31 +60,6 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } -func validateOptions(options *Options) error { - errs := []string{} - if options.URL == "" { - errs = append(errs, "URL") - } - if options.AccountID == "" { - errs = append(errs, "AccountID") - } - if options.Email == "" { - errs = append(errs, "Email") - } - if options.Token == "" { - errs = append(errs, "Token") - } - if options.ProjectName == "" { - errs = append(errs, "ProjectName") - } - - if len(errs) > 0 { - return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) - } - - return nil -} - // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 65958c7af..c2619b7be 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,7 +49,7 @@ type Options struct { // ProjectPath allows nuclei to use a user defined project folder ProjectPath string // InteractshURL is the URL for the interactsh server. - InteractshURL string + InteractshURL string `validate:"omitempty,url"` // Interactsh Authorization header value for self-hosted servers InteractshToken string // Target URLs/Domains to scan using a template diff --git a/v2/pkg/utils/yaml/yaml_decode_wrapper.go b/v2/pkg/utils/yaml/yaml_decode_wrapper.go new file mode 100644 index 000000000..a9cb422bb --- /dev/null +++ b/v2/pkg/utils/yaml/yaml_decode_wrapper.go @@ -0,0 +1,34 @@ +package yaml + +import ( + "io" + "strings" + + "github.com/go-playground/validator/v10" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +var validate *validator.Validate + +// DecodeAndValidate is a wrapper for yaml Decode adding struct validation +func DecodeAndValidate(r io.Reader, v interface{}) error { + if err := yaml.NewDecoder(r).Decode(v); err != nil { + return err + } + if validate == nil { + validate = validator.New() + } + if err := validate.Struct(v); err != nil { + + if _, ok := err.(*validator.InvalidValidationError); ok { + return err + } + errs := []string{} + for _, err := range err.(validator.ValidationErrors) { + errs = append(errs, err.Namespace()+": "+err.Tag()) + } + return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields") + } + return nil +} From 66dacccfb481f3f0bba958b8aee917fc9f5981e1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 20 Nov 2021 19:47:05 +0530 Subject: [PATCH 138/196] Fixed matcher type integer issue --- v2/pkg/operators/matchers/matchers_types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 312d4f59f..f307d08b9 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -106,10 +106,10 @@ func (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error return nil } -func (holder *MatcherTypeHolder) MarshalJSON() ([]byte, error) { - return json.Marshal(holder.MatcherType) +func (holder MatcherTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.MatcherType.String()) } func (holder MatcherTypeHolder) MarshalYAML() (interface{}, error) { - return holder.MatcherType, nil + return holder.MatcherType.String(), nil } From 925fb18855a2ab6de3b5b5565d03d7d6a59c6291 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sat, 20 Nov 2021 14:18:20 +0000 Subject: [PATCH 139/196] Auto Generate Syntax Docs + JSONSchema [Sat Nov 20 14:18:20 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c3fb0b637..6c1742432 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: 1 + - type: word words: - '[core]' - - type: 6 + - type: dsl condition: and dsl: - '!contains(tolower(body), '' Date: Sat, 20 Nov 2021 19:50:18 +0530 Subject: [PATCH 140/196] Fixed jsonschema for matchertype missing String() --- SYNTAX-REFERENCE.md | 16 ++++++++-------- nuclei-jsonschema.json | 12 ++++++------ v2/pkg/operators/matchers/matchers_types.go | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c3fb0b637..6c1742432 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -87,15 +87,15 @@ Examples: ```yaml requests: matchers: - - type: 1 + - type: word words: - '[core]' - - type: 6 + - type: dsl condition: and dsl: - '!contains(tolower(body), '' Date: Mon, 22 Nov 2021 00:19:53 +0100 Subject: [PATCH 141/196] Adding support for clustering within workflow (#1255) * Adding support for clustering within workflow --- DESIGN.md | 3 +- v2/internal/runner/runner.go | 3 +- v2/pkg/core/execute.go | 40 +-------- .../protocols/common/clusterer/clusterer.go | 49 ----------- .../executer.go => templates/cluster.go} | 85 ++++++++++++++++++- v2/pkg/templates/workflows.go | 9 ++ 6 files changed, 95 insertions(+), 94 deletions(-) delete mode 100644 v2/pkg/protocols/common/clusterer/clusterer.go rename v2/pkg/{protocols/common/clusterer/executer.go => templates/cluster.go} (62%) diff --git a/DESIGN.md b/DESIGN.md index f9875b0b6..807852739 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -104,7 +104,7 @@ The `ExecuteWithResults` function accepts a callback, which gets provided with r The default executer is provided in `pkg/protocols/common/executer` . It takes a list of Requests and relevant `ExecuterOptions` and implements the Executer interface required for template execution. The executer during Template compilation process is created from this package and used as-is. -A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/protocols/common/clusterer` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. +A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/templates` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. @@ -585,7 +585,6 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) - [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer - [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers -- [v2/pkg/protocols/common/clusterer](./v2/pkg/protocols/common/clusterer) - HTTP Request Clustering Implementation - [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator - [v2/pkg/protocols/common/helpers/responsehighlighter](./v2/pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter - [v2/pkg/protocols/common/helpers/deserialization](./v2/pkg/protocols/common/helpers/deserialization) - Deserialization helper functions diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9e9ee8f19..9a1d6e716 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -31,6 +31,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" @@ -306,7 +307,7 @@ func (r *Runner) RunEnumeration() error { // Cluster the templates first because we want info on how many // templates did we cluster for showing to user in CLI originalTemplatesCount := len(store.Templates()) - finalTemplates, clusterCount := engine.ClusterTemplates(store.Templates()) + finalTemplates, clusterCount := templates.ClusterTemplates(store.Templates(), engine.ExecuterOptions()) finalTemplates = append(finalTemplates, store.Workflows()...) var totalRequests int64 diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index c9d36cd97..f78400101 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -1,14 +1,10 @@ package core import ( - "fmt" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/remeh/sizedwaitgroup" - "github.com/rs/xid" "go.uber.org/atomic" ) @@ -25,7 +21,7 @@ func (e *Engine) Execute(templates []*templates.Template, target InputProvider) func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { - finalTemplates, _ = e.ClusterTemplates(templatesList) + finalTemplates, _ = templates.ClusterTemplates(templatesList, e.executerOpts) } else { finalTemplates = templatesList } @@ -95,37 +91,3 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template }) wg.Waitgroup.Wait() } - -// ClusterTemplates performs identical http requests clustering for a list of templates -func (e *Engine) ClusterTemplates(templatesList []*templates.Template) ([]*templates.Template, int) { - if e.options.OfflineHTTP { - return templatesList, 0 - } - - templatesMap := make(map[string]*templates.Template) - for _, v := range templatesList { - templatesMap[v.Path] = v - } - clusterCount := 0 - - finalTemplatesList := make([]*templates.Template, 0, len(templatesList)) - clusters := clusterer.Cluster(templatesMap) - for _, cluster := range clusters { - if len(cluster) > 1 { - executerOpts := e.ExecuterOptions() - - clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - - finalTemplatesList = append(finalTemplatesList, &templates.Template{ - ID: clusterID, - RequestsHTTP: cluster[0].RequestsHTTP, - Executer: clusterer.NewExecuter(cluster, &executerOpts), - TotalRequests: len(cluster[0].RequestsHTTP), - }) - clusterCount += len(cluster) - } else { - finalTemplatesList = append(finalTemplatesList, cluster...) - } - } - return finalTemplatesList, clusterCount -} diff --git a/v2/pkg/protocols/common/clusterer/clusterer.go b/v2/pkg/protocols/common/clusterer/clusterer.go deleted file mode 100644 index 29c1a309e..000000000 --- a/v2/pkg/protocols/common/clusterer/clusterer.go +++ /dev/null @@ -1,49 +0,0 @@ -package clusterer - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates" -) - -// Cluster clusters a list of templates into a lesser number if possible based -// on the similarity between the sent requests. -// -// If the attributes match, multiple requests can be clustered into a single -// request which saves time and network resources during execution. -func Cluster(list map[string]*templates.Template) [][]*templates.Template { - final := [][]*templates.Template{} - - // Each protocol that can be clustered should be handled here. - for key, template := range list { - // We only cluster http requests as of now. - // Take care of requests that can't be clustered first. - if len(template.RequestsHTTP) == 0 { - delete(list, key) - final = append(final, []*templates.Template{template}) - continue - } - - delete(list, key) // delete element first so it's not found later. - // Find any/all similar matching request that is identical to - // this one and cluster them together for http protocol only. - if len(template.RequestsHTTP) == 1 { - cluster := []*templates.Template{} - - for otherKey, other := range list { - if len(other.RequestsHTTP) == 0 { - continue - } - if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { - delete(list, otherKey) - cluster = append(cluster, other) - } - } - if len(cluster) > 0 { - cluster = append(cluster, template) - final = append(final, cluster) - continue - } - } - final = append(final, []*templates.Template{template}) - } - return final -} diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/templates/cluster.go similarity index 62% rename from v2/pkg/protocols/common/clusterer/executer.go rename to v2/pkg/templates/cluster.go index c060f341d..7e0ed1cf2 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/templates/cluster.go @@ -1,15 +1,94 @@ -package clusterer +package templates import ( + "fmt" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/rs/xid" ) +// Cluster clusters a list of templates into a lesser number if possible based +// on the similarity between the sent requests. +// +// If the attributes match, multiple requests can be clustered into a single +// request which saves time and network resources during execution. +func Cluster(list map[string]*Template) [][]*Template { + final := [][]*Template{} + + // Each protocol that can be clustered should be handled here. + for key, template := range list { + // We only cluster http requests as of now. + // Take care of requests that can't be clustered first. + if len(template.RequestsHTTP) == 0 { + delete(list, key) + final = append(final, []*Template{template}) + continue + } + + delete(list, key) // delete element first so it's not found later. + // Find any/all similar matching request that is identical to + // this one and cluster them together for http protocol only. + if len(template.RequestsHTTP) == 1 { + cluster := []*Template{} + + for otherKey, other := range list { + if len(other.RequestsHTTP) == 0 { + continue + } + if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + delete(list, otherKey) + cluster = append(cluster, other) + } + } + if len(cluster) > 0 { + cluster = append(cluster, template) + final = append(final, cluster) + continue + } + } + final = append(final, []*Template{template}) + } + return final +} + +func ClusterTemplates(templatesList []*Template, options protocols.ExecuterOptions) ([]*Template, int) { + if options.Options.OfflineHTTP { + return templatesList, 0 + } + + templatesMap := make(map[string]*Template) + for _, v := range templatesList { + templatesMap[v.Path] = v + } + clusterCount := 0 + + finalTemplatesList := make([]*Template, 0, len(templatesList)) + clusters := Cluster(templatesMap) + for _, cluster := range clusters { + if len(cluster) > 1 { + executerOpts := options + + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + finalTemplatesList = append(finalTemplatesList, &Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: NewExecuter(cluster, &executerOpts), + TotalRequests: len(cluster[0].RequestsHTTP), + }) + clusterCount += len(cluster) + } else { + finalTemplatesList = append(finalTemplatesList, cluster...) + } + } + return finalTemplatesList, clusterCount +} + // Executer executes a group of requests for a protocol for a clustered // request. It is different from normal executers since the original // operators are all combined and post processed after making the request. @@ -31,7 +110,7 @@ type clusteredOperator struct { var _ protocols.Executer = &Executer{} // NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOptions) *Executer { +func NewExecuter(requests []*Template, options *protocols.ExecuterOptions) *Executer { executer := &Executer{ options: options, requests: requests[0].RequestsHTTP[0], diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 246c4bf8e..89f8d1bda 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -61,6 +61,9 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr if len(paths) == 0 { return nil } + + var workflowTemplates []*Template + for _, path := range paths { template, err := Parse(path, preprocessor, options.Copy()) if err != nil { @@ -71,10 +74,16 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr gologger.Warning().Msgf("Could not parse workflow template %s: no executer found\n", path) continue } + workflowTemplates = append(workflowTemplates, template) + } + + finalTemplates, _ := ClusterTemplates(workflowTemplates, options.Copy()) + for _, template := range finalTemplates { workflow.Executers = append(workflow.Executers, &workflows.ProtocolExecuterPair{ Executer: template.Executer, Options: options, }) } + return nil } From ba67f8c8fec6b2bd0f939e93f691fa85373a2129 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Nov 2021 09:51:13 +0530 Subject: [PATCH 142/196] fix #1220: no ip returned for dialers --- v2/pkg/protocols/common/protocolstate/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index 0f80f6647..b43e40cf9 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -18,6 +18,7 @@ func Init(options *types.Options) error { if options.ResolversFile != "" { opts.BaseResolvers = options.InternalResolversList } + opts.WithDialerHistory = true dialer, err := fastdialer.NewDialer(opts) if err != nil { return errors.Wrap(err, "could not create dialer") From 1581c96e4e5f90b7c7c4d606be92e6429b3daf92 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 22 Nov 2021 17:53:25 +0530 Subject: [PATCH 143/196] Added matched-status flag + template-path and url to output (#1272) * Added matched-status flag + template-path and url to output --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 2 +- v2/pkg/output/format_screen.go | 15 ++++++- v2/pkg/output/output.go | 41 ++++++++++++++++++- v2/pkg/output/output_test.go | 6 +-- v2/pkg/protocols/common/executer/executer.go | 22 +++++----- .../protocols/common/helpers/writer/writer.go | 35 ++++++++++++++++ .../protocols/common/interactsh/interactsh.go | 16 ++------ v2/pkg/protocols/dns/operators.go | 4 +- v2/pkg/protocols/dns/operators_test.go | 2 +- v2/pkg/protocols/file/operators.go | 4 +- v2/pkg/protocols/file/operators_test.go | 10 ++--- v2/pkg/protocols/headless/operators.go | 4 +- v2/pkg/protocols/http/operators.go | 4 +- v2/pkg/protocols/http/operators_test.go | 8 ++-- v2/pkg/protocols/http/request.go | 3 ++ v2/pkg/protocols/network/operators.go | 4 +- v2/pkg/protocols/network/operators_test.go | 2 +- v2/pkg/protocols/network/request.go | 3 ++ v2/pkg/protocols/offlinehttp/operators.go | 4 +- .../protocols/offlinehttp/operators_test.go | 8 ++-- v2/pkg/protocols/ssl/ssl.go | 4 +- v2/pkg/protocols/websocket/websocket.go | 5 ++- v2/pkg/templates/cluster.go | 25 +++++------ v2/pkg/testutils/testutils.go | 5 +++ v2/pkg/types/types.go | 2 + v2/pkg/utils/template_path.go | 32 +++++++++++++++ 27 files changed, 205 insertions(+), 66 deletions(-) create mode 100644 v2/pkg/protocols/common/helpers/writer/writer.go create mode 100644 v2/pkg/utils/template_path.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 7f24385fd..116479732 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -85,6 +85,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), + flagSet.BoolVarP(&options.MatcherStatus, "matcher-status", "ms", false, "show optional match failure status"), flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a1d6e716..055eca1ab 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -117,7 +117,7 @@ func New(options *types.Options) (*Runner, error) { runner.hmapInputProvider = hmapInput // Create the output file if asked - outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.Output, options.TraceLogFile, options.ErrorLogFile) + outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.Output, options.TraceLogFile, options.ErrorLogFile) if err != nil { return nil, errors.Wrap(err, "could not create output file") } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index ddab852ae..2331095d0 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -27,6 +27,15 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.aurora.BrightGreen(output.ExtractorName).Bold().String()) } + if w.matcherStatus { + builder.WriteString("] [") + if !output.MatcherStatus { + builder.WriteString(w.aurora.Red("failed").String()) + } else { + builder.WriteString(w.aurora.Green("matched").String()) + } + } + builder.WriteString("] [") builder.WriteString(w.aurora.BrightBlue(output.Type).String()) builder.WriteString("] ") @@ -35,7 +44,11 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { builder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity)) builder.WriteString("] ") } - builder.WriteString(output.Matched) + if output.Matched != "" { + builder.WriteString(output.Matched) + } else { + builder.WriteString(output.Host) + } // If any extractors, write the results if len(output.ExtractedResults) > 0 { diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 991c99232..e78699f7a 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" ) @@ -27,6 +28,8 @@ type Writer interface { Colorizer() aurora.Aurora // Write writes the event to file and/or screen. Write(*ResultEvent) error + // WriteFailure writes the optional failure event for template to file and/or screen. + WriteFailure(event InternalEvent) error // Request logs a request in the trace log Request(templateID, url, requestType string, err error) } @@ -37,6 +40,7 @@ type StandardWriter struct { jsonReqResp bool noTimestamp bool noMetadata bool + matcherStatus bool aurora aurora.Aurora outputFile io.WriteCloser traceFile io.WriteCloser @@ -54,10 +58,16 @@ type InternalWrappedEvent struct { InternalEvent InternalEvent Results []*ResultEvent OperatorsResult *operators.Result + UsesInteractsh bool } // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { + // Template is the relative filename for the template + Template string `json:"template,omitempty"` + // TemplateURL is the URL of the template for the result inside the nuclei + // templates repository if it belongs to the repository. + TemplateURL string `json:"template-url,omitempty"` // TemplateID is the ID of the template for the result. TemplateID string `json:"template-id"` // TemplatePath is the path of template @@ -92,12 +102,14 @@ type ResultEvent struct { Interaction *server.Interaction `json:"interaction,omitempty"` // CURLCommand is an optional curl command to reproduce the request // Only applicable if the report is for HTTP. - CURLCommand string `json:"curl-command,omitempty"` + CURLCommand string `json:"curl-command,omitempty"` + // MatcherStatus is the status of the match + MatcherStatus bool `json:"matcher-status"` FileToIndexPosition map[string]int `json:"-"` } // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, file, traceFile string, errorFile string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus bool, file, traceFile string, errorFile string) (*StandardWriter, error) { auroraColorizer := aurora.NewAurora(colors) var outputFile io.WriteCloser @@ -128,6 +140,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, json: json, jsonReqResp: jsonReqResp, noMetadata: noMetadata, + matcherStatus: MatcherStatus, noTimestamp: noTimestamp, aurora: auroraColorizer, outputFile: outputFile, @@ -140,6 +153,10 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp bool, // Write writes the event to file and/or screen. func (w *StandardWriter) Write(event *ResultEvent) error { + // Enrich the result event with extra metadata on the template-path and url. + if event.TemplatePath != "" { + event.Template, event.TemplateURL = utils.TemplatePathURL(types.ToString(event.TemplatePath)) + } event.Timestamp = time.Now() var data []byte @@ -224,3 +241,23 @@ func (w *StandardWriter) Close() { w.errorFile.Close() } } + +// WriteFailure writes the failure event for template to file and/or screen. +func (w *StandardWriter) WriteFailure(event InternalEvent) error { + if !w.matcherStatus { + return nil + } + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"])) + data := &ResultEvent{ + Template: templatePath, + TemplateURL: templateURL, + TemplateID: types.ToString(event["template-id"]), + TemplatePath: types.ToString(event["template-path"]), + Info: event["template-info"].(model.Info), + Type: types.ToString(event["type"]), + Host: types.ToString(event["host"]), + MatcherStatus: false, + Timestamp: time.Now(), + } + return w.Write(data) +} diff --git a/v2/pkg/output/output_test.go b/v2/pkg/output/output_test.go index 9f1d188bb..1648dcc36 100644 --- a/v2/pkg/output/output_test.go +++ b/v2/pkg/output/output_test.go @@ -11,7 +11,7 @@ import ( func TestStandardWriterRequest(t *testing.T) { t.Run("WithoutTraceAndError", func(t *testing.T) { - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") require.NoError(t, err) require.NotPanics(t, func() { w.Request("path", "input", "http", nil) @@ -23,7 +23,7 @@ func TestStandardWriterRequest(t *testing.T) { traceWriter := &testWriteCloser{} errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.traceFile = traceWriter w.errorFile = errorWriter require.NoError(t, err) @@ -36,7 +36,7 @@ func TestStandardWriterRequest(t *testing.T) { t.Run("ErrorWithWrappedError", func(t *testing.T) { errorWriter := &testWriteCloser{} - w, err := NewStandardWriter(false, false, false, false, false, "", "", "") + w, err := NewStandardWriter(false, false, false, false, false, false, "", "", "") w.errorFile = errorWriter require.NoError(t, err) w.Request( diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 3aaf774cb..fe650e945 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" ) // Executer executes a group of requests for a protocol @@ -59,18 +60,17 @@ func (e *Executer) Execute(input string) (bool, error) { builder.Reset() } } - if event.OperatorsResult == nil { - return - } - for _, result := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if event.OperatorsResult == nil && !event.UsesInteractsh { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + } else { + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results = true } - results = true - _ = e.options.Output.Write(result) - e.options.Progress.IncrementMatched() } }) if err != nil { diff --git a/v2/pkg/protocols/common/helpers/writer/writer.go b/v2/pkg/protocols/common/helpers/writer/writer.go new file mode 100644 index 000000000..91b98f33b --- /dev/null +++ b/v2/pkg/protocols/common/helpers/writer/writer.go @@ -0,0 +1,35 @@ +package writer + +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" +) + +// WriteResult is a helper for writing results to the output +func WriteResult(data *output.InternalWrappedEvent, output output.Writer, progress progress.Progress, issuesClient *reporting.Client) bool { + // Handle the case where no result found for the template. + // In this case, we just show misc information about the failed + // match for the template. + if data.OperatorsResult == nil { + return false + } + var matched bool + for _, result := range data.Results { + if err := output.Write(result); err != nil { + gologger.Warning().Msgf("Could not write output event: %s\n", err) + } + if !matched { + matched = true + } + progress.IncrementMatched() + + if issuesClient != nil { + if err := issuesClient.CreateIssue(result); err != nil { + gologger.Warning().Msgf("Could not create issue on tracker: %s", err) + } + } + } + return matched +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 7c3e02f82..b810c346d 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" ) @@ -177,19 +178,8 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d } data.Event.Results = data.MakeResultFunc(data.Event) - for _, result := range data.Event.Results { - result.Interaction = interaction - _ = c.options.Output.Write(result) - if !c.matched { - c.matched = true - } - c.options.Progress.IncrementMatched() - - if c.options.IssuesClient != nil { - if err := c.options.IssuesClient.CreateIssue(result); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } + if writer.WriteResult(data.Event, c.options.Output, c.options.Progress, c.options.IssuesClient) { + c.matched = true } return true } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 12de136a8..0c1ff929c 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -90,6 +90,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, + "type": request.Type().String(), "trace": traceToString(tracedata, false), } } @@ -104,10 +105,11 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "dns", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, Timestamp: time.Now(), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index aef43ccfa..5651ea422 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -45,7 +45,7 @@ func TestResponseToDSLMap(t *testing.T) { resp.Answer = append(resp.Answer, &dns.A{A: net.ParseIP("1.1.1.1"), Hdr: dns.RR_Header{Name: "one.one.one.one."}}) event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", nil) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, dns.RcodeSuccess, event["rcode"], "could not get correct rcode") } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index cde2e63e0..41ef7fd3a 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName str "path": inputFilePath, "matched": matchedFileName, "raw": raw, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -120,10 +121,11 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ + MatcherStatus: true, TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "file", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index ac41a9400..ffdddc083 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { resp := "test-data\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") } @@ -60,7 +60,7 @@ func TestFileOperatorMatch(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("valid", func(t *testing.T) { @@ -109,7 +109,7 @@ func TestFileOperatorMatch(t *testing.T) { t.Run("caseInsensitive", func(t *testing.T) { resp := "TEST-DATA\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") matcher := &matchers.Matcher{ @@ -148,7 +148,7 @@ func TestFileOperatorExtract(t *testing.T) { resp := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["raw"], "could not get correct resp") t.Run("extract", func(t *testing.T) { @@ -266,7 +266,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi fileContent := "test-data\r\n1.1.1.1\r\n" event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) - require.Len(t, event, 6, "could not get correct number of items in dsl map") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 3d62e61b5..b9f9b4ccc 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -72,6 +72,7 @@ func (request *Request) responseToDSLMap(resp, req, host, matched string) output "matched": matched, "req": req, "data": resp, + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -92,11 +93,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "headless", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 868fd0118..1f319ccff 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data[k] = strings.Join(v, " ") } data["host"] = host + data["type"] = request.Type().String() data["matched"] = matched data["request"] = rawReq data["response"] = rawResp @@ -141,12 +142,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 7aefd7d19..ae6c644cf 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -41,7 +41,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -71,7 +71,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -159,7 +159,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test_header"], "could not get correct resp for header") @@ -286,7 +286,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 94782c2a9..8f2040782 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -473,6 +473,9 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) + if hasInteractMarkers { + event.UsesInteractsh = true + } responseContentType := resp.Header.Get("Content-Type") dumpResponse(event, request.options, response.fullResponse, formedURL, responseContentType) diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c829b8ade..010b11d61 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -73,6 +73,7 @@ func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) o "request": req, "data": resp, // Data is the last bytes read "raw": raw, // Raw is the full transaction data for network + "type": request.Type().String(), "template-id": request.options.TemplateID, "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, @@ -93,12 +94,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "network", + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Metadata: wrapped.OperatorsResult.PayloadValues, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["data"]), diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index daa92725e..805c0ed22 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -35,7 +35,7 @@ func TestResponseToDSLMap(t *testing.T) { req := "test-data\r\n" resp := "resp-data\r\n" event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one") - require.Len(t, event, 8, "could not get correct number of items in dsl map") + require.Len(t, event, 9, "could not get correct number of items in dsl map") require.Equal(t, resp, event["data"], "could not get correct resp") } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index ae63dc0d0..35da81d15 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -281,6 +281,9 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input ExtractFunc: request.Extract, }) } + if len(interactshURLs) > 0 { + event.UsesInteractsh = true + } dumpResponse(event, request.options, response, actualAddress) diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index b5141037b..203693090 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -113,6 +113,7 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body + data["type"] = request.Type().String() data["all_headers"] = headers data["duration"] = duration.Seconds() data["template-id"] = request.options.TemplateID @@ -135,11 +136,12 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(model.Info), - Type: "http", + Type: types.ToString(wrapped.InternalEvent["type"]), Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["raw"]), diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index b69ccbdc2..0a3a663f0 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -37,7 +37,7 @@ func TestResponseToDSLMap(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") } @@ -63,7 +63,7 @@ func TestHTTPOperatorMatch(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") @@ -132,7 +132,7 @@ func TestHTTPOperatorExtract(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test-header"], "could not get correct resp for header") @@ -198,7 +198,7 @@ func TestHTTPMakeResult(t *testing.T) { matched := "http://example.com/test/?test=1" event := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{}) - require.Len(t, event, 13, "could not get correct number of items in dsl map") + require.Len(t, event, 14, "could not get correct number of items in dsl map") require.Equal(t, exampleRawResponse, event["response"], "could not get correct resp") require.Equal(t, "Test-Response", event["test"], "could not get correct resp for header") diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index ba26c8e30..388886be2 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -129,6 +129,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous data := make(map[string]interface{}) cert := connTLS.ConnectionState().PeerCertificates[0] + data["type"] = request.Type().String() data["response"] = jsonDataString data["host"] = input data["matched"] = addressToDial @@ -195,12 +196,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["host"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), } return data diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 47a87a78d..ab864f65b 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -248,6 +248,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam for k, v := range events { data[k] = v } + + data["type"] = request.Type().String() data["success"] = "true" data["request"] = requestOutput data["response"] = responseBuilder.String() @@ -364,12 +366,13 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent TemplateID: types.ToString(request.options.TemplateID), TemplatePath: types.ToString(request.options.TemplatePath), Info: request.options.TemplateInfo, - Type: request.Type().String(), + Type: types.ToString(wrapped.InternalEvent["type"]), Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), + MatcherStatus: true, IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 7e0ed1cf2..1ec1e95cb 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/rs/xid" ) @@ -147,22 +148,22 @@ func (e *Executer) Execute(input string) (bool, error) { err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) + event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath + event.InternalEvent["template-info"] = operator.templateInfo + + if result == nil && !matched { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + continue + } if matched && result != nil { event.OperatorsResult = result - event.InternalEvent["template-id"] = operator.templateID - event.InternalEvent["template-path"] = operator.templatePath - event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) results = true - for _, r := range event.Results { - if e.options.IssuesClient != nil { - if err := e.options.IssuesClient.CreateIssue(r); err != nil { - gologger.Warning().Msgf("Could not create issue on tracker: %s", err) - } - } - _ = e.options.Output.Write(r) - e.options.Progress.IncrementMatched() - } + + _ = writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) } } }) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 8e0c5923f..982fbabc3 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -131,6 +131,11 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro } } +// Write writes the event to file and/or screen. +func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { + return nil +} + type MockProgressClient struct{} // Stop stops the progress recorder. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index c2619b7be..1a0935338 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -177,6 +177,8 @@ type Options struct { NoUpdateTemplates bool // EnvironmentVariables enables support for environment variables EnvironmentVariables bool + // MatcherStatus displays optional status for the failed matches as well + MatcherStatus bool // ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts ClientCertFile string // ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts diff --git a/v2/pkg/utils/template_path.go b/v2/pkg/utils/template_path.go new file mode 100644 index 000000000..2039c2d97 --- /dev/null +++ b/v2/pkg/utils/template_path.go @@ -0,0 +1,32 @@ +package utils + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" +) + +const ( + // TemplatesRepoURL is the URL for files in nuclei-templates repository + TemplatesRepoURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master/" +) + +var configData *config.Config + +func init() { + configData, _ = config.ReadConfiguration() +} + +// TemplatePathURL returns the Path and URL for the provided template +func TemplatePathURL(fullPath string) (string, string) { + var templateDirectory string + if configData != nil && configData.TemplatesDirectory != "" && strings.HasPrefix(fullPath, configData.TemplatesDirectory) { + templateDirectory = configData.TemplatesDirectory + } else { + return "", "" + } + + finalPath := strings.TrimPrefix(strings.TrimPrefix(fullPath, templateDirectory), "/") + templateURL := TemplatesRepoURL + finalPath + return finalPath, templateURL +} From 6c819d7917f2779f32cce266ed6e56ea783df1e2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 10:44:31 +0530 Subject: [PATCH 144/196] Added additional variables for network + simplified logic --- v2/pkg/protocols/network/network.go | 16 ++------- v2/pkg/protocols/network/network_test.go | 15 ++------- v2/pkg/protocols/network/request.go | 41 ++++++++++++++---------- v2/pkg/protocols/network/request_test.go | 9 ++---- 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index a5274eff6..4495d0b03 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -2,7 +2,6 @@ package network import ( "fmt" - "net" "strings" "github.com/pkg/errors" @@ -78,9 +77,8 @@ type Request struct { } type addressKV struct { - ip string - port string - tls bool + address string + tls bool } // Input is the input to send on the network @@ -136,15 +134,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { shouldUseTLS = true address = strings.TrimPrefix(address, "tls://") } - if strings.Contains(address, ":") { - addressHost, addressPort, portErr := net.SplitHostPort(address) - if portErr != nil { - return errors.Wrap(portErr, "could not parse address") - } - request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) - } else { - request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) - } + request.addresses = append(request.addresses, addressKV{address: address, tls: shouldUseTLS}) } // Pre-compile any input dsl functions before executing the request. for _, input := range request.Inputs { diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 180b34192..733211a9e 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"{{Hostname}}", "{{Hostname}}:8082", "tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Hostname}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } @@ -28,17 +28,8 @@ func TestNetworkCompileMake(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile network request") - require.Equal(t, 3, len(request.addresses), "could not get correct number of input address") - t.Run("check-host", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[0].ip, "could not get correct host") - }) - t.Run("check-host-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[1].ip, "could not get correct host with port") - require.Equal(t, "8082", request.addresses[1].port, "could not get correct port for host") - }) + require.Equal(t, 1, len(request.addresses), "could not get correct number of input address") t.Run("check-tls-with-port", func(t *testing.T) { - require.Equal(t, "{{Hostname}}", request.addresses[2].ip, "could not get correct host with port") - require.Equal(t, "443", request.addresses[2].port, "could not get correct port for host") - require.True(t, request.addresses[2].tls, "could not get correct port for host") + require.True(t, request.addresses[0].tls, "could not get correct port for host") }) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index acbbac9af..2446bc4db 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -41,18 +41,10 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } for _, kv := range request.addresses { - actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) - if kv.port != "" { - if strings.Contains(address, ":") { - actualAddress, _, _ = net.SplitHostPort(actualAddress) - } - actualAddress = net.JoinHostPort(actualAddress, kv.port) - } - if input != "" { - input = actualAddress - } + variables := generateNetworkVariables(address) + actualAddress := replacer.Replace(kv.address, variables) - if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -61,7 +53,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } // executeAddress executes the request for an address -func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") request.options.Output.Request(request.options.TemplateID, address, "network", err) @@ -80,27 +72,27 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho break } value = generators.MergeMaps(value, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := generators.MergeMaps(map[string]interface{}{}, payloads) - if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + value := generators.CopyMap(payloads) + if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) - request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + request.dynamicValues = generators.MergeMaps(payloads, variables) if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host @@ -257,3 +249,18 @@ func getAddress(toTest string) (string, error) { } return toTest, nil } + +func generateNetworkVariables(input string) map[string]interface{} { + if !strings.Contains(input, ":") { + return map[string]interface{}{"Hostname": input, "Host": input} + } + host, port, err := net.SplitHostPort(input) + if err != nil { + return map[string]interface{}{"Hostname": input} + } + return map[string]interface{}{ + "Host": host, + "Port": port, + "Hostname": input, + } +} diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index d1f266b48..4cc578919 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -50,7 +50,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse url") - request.Address[0] = "{{Hostname}}:" + parsed.Port() + request.Address[0] = "{{Hostname}}" request.Inputs = append(request.Inputs, &Input{Data: fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host)}) executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -84,12 +84,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.NotNil(t, finalEvent, "could not get event output from request") - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results") - require.Equal(t, "

Example Domain

", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") - finalEvent = nil + require.Nil(t, finalEvent, "could not get event output from request") request.Inputs[0].Type = "hex" request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) From 831114bdbd51a94228290e82af7a1c02c61e86aa Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:09:22 +0530 Subject: [PATCH 145/196] Added default fields for DNS requests --- v2/pkg/protocols/dns/dns.go | 14 ++++++++++++-- v2/pkg/protocols/dns/dns_test.go | 3 ++- v2/pkg/protocols/dns/operators_test.go | 12 ++++++++---- v2/pkg/protocols/dns/request_test.go | 3 ++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 7cde7fed4..177a497b7 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -83,7 +83,7 @@ type Request struct { // description: | // Recursion determines if resolver should recurse all records to get fresh results. - Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` + Recursion *bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` // Resolvers to use for the dns requests Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } @@ -99,6 +99,12 @@ func (request *Request) GetID() string { // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if request.Retries == 0 { + request.Retries = 3 + } + if request.Recursion == nil { + *request.Recursion = true + } dnsClientOptions := &dnsclientpool.Configuration{ Retries: request.Retries, } @@ -162,7 +168,7 @@ func (request *Request) Make(domain string) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() - req.RecursionDesired = request.Recursion + req.RecursionDesired = *request.Recursion var q dns.Question @@ -207,6 +213,8 @@ func questionTypeToInt(questionType string) uint16 { question = dns.TypeDS case "AAAA": question = dns.TypeAAAA + default: + question = dns.TypeA } return question } @@ -229,6 +237,8 @@ func classToInt(class string) uint16 { result = dns.ClassNONE case "ANY": result = dns.ClassANY + default: + result = dns.ClassINET } return uint16(result) } diff --git a/v2/pkg/protocols/dns/dns_test.go b/v2/pkg/protocols/dns/dns_test.go index f44ba2205..0aae3ed20 100644 --- a/v2/pkg/protocols/dns/dns_test.go +++ b/v2/pkg/protocols/dns/dns_test.go @@ -24,6 +24,7 @@ func TestGenerateDNSVariables(t *testing.T) { func TestDNSCompileMake(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) const templateID = "testing-dns" request := &Request{ @@ -31,7 +32,7 @@ func TestDNSCompileMake(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 5651ea422..963f6bd8d 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -20,6 +20,7 @@ import ( func TestResponseToDSLMap(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -27,7 +28,7 @@ func TestResponseToDSLMap(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -52,6 +53,7 @@ func TestResponseToDSLMap(t *testing.T) { func TestDNSOperatorMatch(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -59,7 +61,7 @@ func TestDNSOperatorMatch(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -163,6 +165,7 @@ func TestDNSOperatorMatch(t *testing.T) { func TestDNSOperatorExtract(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -170,7 +173,7 @@ func TestDNSOperatorExtract(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ @@ -220,6 +223,7 @@ func TestDNSOperatorExtract(t *testing.T) { func TestDNSMakeResult(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -227,7 +231,7 @@ func TestDNSMakeResult(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 7b6a3e6ab..e425939ee 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -17,6 +17,7 @@ import ( func TestDNSExecuteWithResults(t *testing.T) { options := testutils.DefaultOptions + recursion := false testutils.Init(options) templateID := "testing-dns" request := &Request{ @@ -24,7 +25,7 @@ func TestDNSExecuteWithResults(t *testing.T) { Class: "INET", Retries: 5, ID: templateID, - Recursion: false, + Recursion: &recursion, Name: "{{FQDN}}", Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ From 44e304179a4a4e82593798d7c97ca25b1168c689 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:15:24 +0530 Subject: [PATCH 146/196] Fixed a linter error --- v2/pkg/protocols/dns/dns.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 177a497b7..c7dbd3134 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -213,8 +213,6 @@ func questionTypeToInt(questionType string) uint16 { question = dns.TypeDS case "AAAA": question = dns.TypeAAAA - default: - question = dns.TypeA } return question } @@ -237,8 +235,6 @@ func classToInt(class string) uint16 { result = dns.ClassNONE case "ANY": result = dns.ClassANY - default: - result = dns.ClassINET } return uint16(result) } From a908a1515aa3a704f5dd614cd580a50614901c02 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 23 Nov 2021 13:17:19 +0530 Subject: [PATCH 147/196] Fixed example docs --- v2/pkg/templates/templates_doc_examples.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 49e58eb00..aa8a47a1f 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -36,12 +36,13 @@ var ( } _ = exampleNormalHTTPRequest + recursion = false exampleNormalDNSRequest = &dns.Request{ Name: "{{FQDN}}", RequestType: dns.DNSRequestTypeHolder{DNSRequestType: dns.CNAME}, Class: "inet", Retries: 2, - Recursion: true, + Recursion: &recursion, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, From ef5b476c6dff104e826b46dd402dff9130ab4132 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 24 Nov 2021 21:08:08 +0530 Subject: [PATCH 148/196] Reusing dynamically extracted values as iterators in http request This PR adds the support in http module to iterate over the dynamically extracted data from extractors and use it in other requests. This allows nuclei to follow links on pages, do operations with multiple versions of the same extracted value, etc. --- v2/pkg/operators/operators.go | 53 ++++++- v2/pkg/operators/operators_test.go | 39 +++++ v2/pkg/protocols/common/generators/maps.go | 32 +++++ v2/pkg/protocols/http/build_request.go | 16 +-- v2/pkg/protocols/http/build_request_test.go | 24 ++-- v2/pkg/protocols/http/request.go | 150 +++++++++++++------- v2/pkg/protocols/http/request_test.go | 93 ++++++++++++ 7 files changed, 329 insertions(+), 78 deletions(-) create mode 100644 v2/pkg/operators/operators_test.go create mode 100644 v2/pkg/protocols/http/request_test.go diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 0b9087312..6a88e0785 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -72,11 +72,54 @@ type Result struct { // OutputExtracts is the list of extracts to be displayed on screen. OutputExtracts []string // DynamicValues contains any dynamic values to be templated - DynamicValues map[string]interface{} + DynamicValues map[string][]string // PayloadValues contains payload values provided by user. (Optional) PayloadValues map[string]interface{} } +// MakeDynamicValuesCallback takes an input dynamic values map and calls +// the callback function with all variations of the data in input in form +// of map[string]string (interface{}). +func MakeDynamicValuesCallback(input map[string][]string, callback func(map[string]interface{}) bool) { + output := make(map[string]interface{}, len(input)) + inputIndex := make(map[string]int, len(input)) + + var maxValue int + for _, v := range input { + if len(v) > maxValue { + maxValue = len(v) + } + } + + for i := 0; i < maxValue; i++ { + for k, v := range input { + if len(v) == 0 { + continue + } + if len(v) == 1 { + output[k] = v[0] + continue + } + if gotIndex, ok := inputIndex[k]; !ok { + inputIndex[k] = 0 + output[k] = v[0] + } else { + newIndex := gotIndex + 1 + if newIndex >= len(v) { + output[k] = v[len(v)-1] + continue + } + output[k] = v[newIndex] + inputIndex[k] = newIndex + } + } + // skip if the callback says so + if callback(output) { + return + } + } +} + // Merge merges a result structure into the other. func (r *Result) Merge(result *Result) { if !r.Matched && result.Matched { @@ -115,7 +158,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc result := &Result{ Matches: make(map[string][]string), Extracts: make(map[string][]string), - DynamicValues: make(map[string]interface{}), + DynamicValues: make(map[string][]string), } // Start with the extractors first and evaluate them. @@ -126,8 +169,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc extractorResults = append(extractorResults, match) if extractor.Internal { - if _, ok := result.DynamicValues[extractor.Name]; !ok { - result.DynamicValues[extractor.Name] = match + if data, ok := result.DynamicValues[extractor.Name]; !ok { + result.DynamicValues[extractor.Name] = []string{match} + } else { + result.DynamicValues[extractor.Name] = append(data, match) } } else { result.OutputExtracts = append(result.OutputExtracts, match) diff --git a/v2/pkg/operators/operators_test.go b/v2/pkg/operators/operators_test.go new file mode 100644 index 000000000..cb000849b --- /dev/null +++ b/v2/pkg/operators/operators_test.go @@ -0,0 +1,39 @@ +package operators + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMakeDynamicValuesCallback(t *testing.T) { + input := map[string][]string{ + "a": []string{"1", "2"}, + "b": []string{"3"}, + "c": []string{}, + "d": []string{"A", "B", "C"}, + } + + count := 0 + MakeDynamicValuesCallback(input, func(data map[string]interface{}) bool { + count++ + require.Len(t, data, 3, "could not get correct output length") + return false + }) + require.Equal(t, 3, count, "could not get correct result count") + + t.Run("single", func(t *testing.T) { + input := map[string][]string{ + "a": []string{"1"}, + "b": []string{"2"}, + "c": []string{"3"}, + } + + count := 0 + MakeDynamicValuesCallback(input, func(data map[string]interface{}) { + count++ + require.Len(t, data, 3, "could not get correct output length") + }) + require.Equal(t, 1, count, "could not get correct result count") + }) +} diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index 31df768bb..4660f721d 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -1,9 +1,41 @@ package generators import ( + "reflect" "strings" ) +// MergeMapsMany merges many maps into a new map +func MergeMapsMany(maps ...interface{}) map[string][]string { + m := make(map[string][]string) + for _, gotMap := range maps { + val := reflect.ValueOf(gotMap) + if val.Kind() != reflect.Map { + continue + } + appendToSlice := func(key, value reflect.Value) { + keyStr, valueStr := key.String(), value.String() + if values, ok := m[keyStr]; !ok { + m[keyStr] = []string{valueStr} + } else { + m[keyStr] = append(values, valueStr) + } + } + for _, e := range val.MapKeys() { + v := val.MapIndex(e) + switch v.Kind() { + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + appendToSlice(e, v.Index(i)) + } + case reflect.String: + appendToSlice(e, v) + } + } + } + return m +} + // MergeMaps merges two maps into a new map func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} { m := make(map[string]interface{}, len(m1)+len(m2)) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc8..568572424 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -53,14 +53,9 @@ func (g *generatedRequest) URL() string { // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. -func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) { +func (r *requestGenerator) Make(baseURL, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { if r.request.SelfContained { - return r.makeSelfContainedRequest(dynamicValues) - } - // We get the next payload for the request. - data, payloads, ok := r.nextValue() - if !ok { - return nil, io.EOF + return r.makeSelfContainedRequest(data, payloads, dynamicValues) } ctx := context.Background() @@ -107,12 +102,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values, payloads) } -func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}) (*generatedRequest, error) { - // We get the next payload for the request. - data, payloads, ok := r.nextValue() - if !ok { - return nil, io.EOF - } +func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { ctx := context.Background() isRawRequest := r.request.isRaw() diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index a77e3c947..3e323a4db 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -86,7 +86,8 @@ func TestMakeRequestFromModal(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ := generator.nextValue() + req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") bodyBytes, _ := req.request.BodyBytes() @@ -113,12 +114,14 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) { require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com/test.php", map[string]interface{}{}) + inputData, payloads, _ := generator.nextValue() + req, err := generator.Make("https://example.com/test.php", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path") generator = request.newGenerator() - req, err = generator.Make("https://example.com/test/", map[string]interface{}{}) + inputData, payloads, _ = generator.nextValue() + req, err = generator.Make("https://example.com/test/", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path") } @@ -151,12 +154,14 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ := generator.nextValue() + req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ = generator.nextValue() + req, err = generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw") @@ -190,12 +195,14 @@ Accept-Encoding: gzip`}, require.Nil(t, err, "could not compile http request") generator := request.newGenerator() - req, err := generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ := generator.nextValue() + req, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization := req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw") - req, err = generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ = generator.nextValue() + req, err = generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") authorization = req.request.Header.Get("Authorization") require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw") @@ -231,7 +238,8 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { }) require.Nil(t, err, "could not create interactsh client") - got, err := generator.Make("https://example.com", map[string]interface{}{}) + inputData, payloads, _ := generator.nextValue() + got, err := generator.Make("https://example.com", inputData, payloads, map[string]interface{}{}) require.Nil(t, err, "could not make http request") // check if all the interactsh markers are replaced with unique urls diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..2c7dc0058 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -19,6 +19,7 @@ import ( "moul.io/http2curl" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -47,7 +48,12 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent generator := request.newGenerator() - requestForDump, err := generator.Make(reqURL, nil) + + inputData, payloads, ok := generator.nextValue() + if !ok { + return nil + } + requestForDump, err := generator.Make(reqURL, inputData, payloads, nil) if err != nil { return err } @@ -65,7 +71,11 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern // Pre-Generate requests for i := 0; i < request.RaceNumberRequests; i++ { generator := request.newGenerator() - generatedRequest, err := generator.Make(reqURL, nil) + inputData, payloads, ok := generator.nextValue() + if !ok { + break + } + generatedRequest, err := generator.Make(reqURL, inputData, payloads, nil) if err != nil { return err } @@ -104,7 +114,11 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) + inputData, payloads, ok := generator.nextValue() + if !ok { + break + } + generatedHttpRequest, err := generator.Make(reqURL, inputData, payloads, dynamicValues) if err != nil { if err == io.EOF { break @@ -167,11 +181,12 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous var requestErr error mutex := &sync.Mutex{} for { - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) + inputData, payloads, ok := generator.nextValue() + if !ok { + break + } + generatedHttpRequest, err := generator.Make(reqURL, inputData, payloads, dynamicValues) if err != nil { - if err == io.EOF { - break - } request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } @@ -215,62 +230,91 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou generator := request.newGenerator() + var gotDynamicValues map[string][]string requestCount := 1 var requestErr error for { - hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) + // returns two values, error and skip, which skips the execution for the request instance. + executeFunc := func(data string, payloads, dynamicValue map[string]interface{}) (bool, error) { + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) - generatedHttpRequest, err := generator.Make(reqURL, dynamicValues) - if err != nil { - if err == io.EOF { - break + generatedHttpRequest, err := generator.Make(reqURL, data, payloads, dynamicValue) + if err != nil { + if err == io.EOF { + return true, nil + } + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + return true, err } - request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) - return err + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } + request.dynamicValues = generatedHttpRequest.dynamicValues + // Check if hosts just keep erroring + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { + return true, nil + } + var gotOutput bool + request.options.RateLimiter.Take() + + err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotOutput = true + gotDynamicValues = generators.MergeMapsMany(event.OperatorsResult.DynamicValues, dynamicValues, gotDynamicValues) + } + if hasInteractMarkers && request.options.Interactsh != nil { + request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, + Event: event, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, + }) + } else { + callback(event) + } + }, requestCount) + // If a variable is unresolved, skip all further requests + if err == errStopExecution { + return true, nil + } + if err != nil { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) { + request.options.HostErrorsCache.MarkFailed(reqURL) + } + requestErr = err + } + requestCount++ + request.options.Progress.IncrementRequests() + + // If this was a match and we want to stop at first match, skip all further requests. + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { + return true, nil + } + return false, nil } - if reqURL == "" { - reqURL = generatedHttpRequest.URL() - } - request.dynamicValues = generatedHttpRequest.dynamicValues - // Check if hosts just keep erroring - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { + + inputData, payloads, ok := generator.nextValue() + if !ok { break } - var gotOutput bool - request.options.RateLimiter.Take() - err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { - // Add the extracts to the dynamic values if any. - if event.OperatorsResult != nil { - gotOutput = true - dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) - } - if hasInteractMarkers && request.options.Interactsh != nil { - request.options.Interactsh.RequestEvent(generatedHttpRequest.interactshURLs, &interactsh.RequestData{ - MakeResultFunc: request.MakeResultEvent, - Event: event, - Operators: request.CompiledOperators, - MatchFunc: request.Match, - ExtractFunc: request.Extract, - }) - } else { - callback(event) - } - }, requestCount) - // If a variable is unresolved, skip all further requests - if err == errStopExecution { - break + var gotErr error + var skip bool + if len(gotDynamicValues) > 0 { + operators.MakeDynamicValuesCallback(gotDynamicValues, func(data map[string]interface{}) bool { + if skip, gotErr = executeFunc(inputData, payloads, data); skip || gotErr != nil { + return true + } + return false + }) + } else { + skip, gotErr = executeFunc(inputData, payloads, dynamicValues) } - if err != nil { - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) { - request.options.HostErrorsCache.MarkFailed(reqURL) - } - requestErr = err + if gotErr != nil && requestErr == nil { + requestErr = gotErr } - requestCount++ - request.options.Progress.IncrementRequests() - - // If this was a match and we want to stop at first match, skip all further requests. - if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { + if skip || gotErr != nil { break } } diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go new file mode 100644 index 000000000..12302a229 --- /dev/null +++ b/v2/pkg/protocols/http/request_test.go @@ -0,0 +1,93 @@ +package http + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +func TestHTTPExtractMultipleReuse(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-http" + request := &Request{ + ID: templateID, + Raw: []string{ + `GET /robots.txt HTTP/1.1 + Host: {{Hostname}} + User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Language: en-US,en;q=0.5 + `, + + `GET {{endpoint}} HTTP/1.1 + Host: {{Hostname}} + User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Language: en-US,en;q=0.5 + `, + }, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Part: "body", + Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher}, + Words: []string{"match /a", "match /b", "match /c"}, + }}, + Extractors: []*extractors.Extractor{{ + Part: "body", + Name: "endpoint", + Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Regex: []string{"(?m)/([a-zA-Z0-9-_/\\\\]+)"}, + Internal: true, + }}, + }, + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/robots.txt": + w.Write([]byte(`User-agent: Googlebot +Disallow: /a +Disallow: /b +Disallow: /c`)) + default: + w.Write([]byte(fmt.Sprintf(`match %v`, r.URL.Path))) + } + })) + defer ts.Close() + + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + var finalEvent *output.InternalWrappedEvent + var matchCount int + t.Run("test", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + err := request.ExecuteWithResults(ts.URL, metadata, previous, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult != nil && event.OperatorsResult.Matched { + matchCount++ + } + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 3, matchCount, "could not get correct match count") +} From 273233bb818d7973a73129ce2fec177a207f91d4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 24 Nov 2021 21:56:55 +0530 Subject: [PATCH 149/196] Fixed a bug with mergemanymaps + misc lint fixes --- v2/pkg/operators/operators_test.go | 3 ++- v2/pkg/protocols/common/generators/maps.go | 25 +++++++++++++------ .../protocols/common/generators/maps_test.go | 16 ++++++++++++ v2/pkg/protocols/http/request_test.go | 4 +-- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/maps_test.go diff --git a/v2/pkg/operators/operators_test.go b/v2/pkg/operators/operators_test.go index cb000849b..023a6c1f0 100644 --- a/v2/pkg/operators/operators_test.go +++ b/v2/pkg/operators/operators_test.go @@ -30,9 +30,10 @@ func TestMakeDynamicValuesCallback(t *testing.T) { } count := 0 - MakeDynamicValuesCallback(input, func(data map[string]interface{}) { + MakeDynamicValuesCallback(input, func(data map[string]interface{}) bool { count++ require.Len(t, data, 3, "could not get correct output length") + return false }) require.Equal(t, 1, count, "could not get correct result count") }) diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index 4660f721d..167dc85a9 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -1,6 +1,7 @@ package generators import ( + "fmt" "reflect" "strings" ) @@ -13,12 +14,11 @@ func MergeMapsMany(maps ...interface{}) map[string][]string { if val.Kind() != reflect.Map { continue } - appendToSlice := func(key, value reflect.Value) { - keyStr, valueStr := key.String(), value.String() - if values, ok := m[keyStr]; !ok { - m[keyStr] = []string{valueStr} + appendToSlice := func(key, value string) { + if values, ok := m[key]; !ok { + m[key] = []string{value} } else { - m[keyStr] = append(values, valueStr) + m[key] = append(values, value) } } for _, e := range val.MapKeys() { @@ -26,10 +26,21 @@ func MergeMapsMany(maps ...interface{}) map[string][]string { switch v.Kind() { case reflect.Slice, reflect.Array: for i := 0; i < v.Len(); i++ { - appendToSlice(e, v.Index(i)) + appendToSlice(e.String(), v.Index(i).String()) } case reflect.String: - appendToSlice(e, v) + appendToSlice(e.String(), v.String()) + case reflect.Interface: + switch data := v.Interface().(type) { + case string: + appendToSlice(e.String(), data) + case []string: + for _, value := range data { + appendToSlice(e.String(), value) + } + } + default: + fmt.Printf("invalid type: %v\n", v.Kind()) } } } diff --git a/v2/pkg/protocols/common/generators/maps_test.go b/v2/pkg/protocols/common/generators/maps_test.go new file mode 100644 index 000000000..870af84c9 --- /dev/null +++ b/v2/pkg/protocols/common/generators/maps_test.go @@ -0,0 +1,16 @@ +package generators + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMergeMapsMany(t *testing.T) { + got := MergeMapsMany(map[string]interface{}{"a": []string{"1", "2"}, "c": "5"}, map[string][]string{"b": []string{"3", "4"}}) + require.Equal(t, map[string][]string{ + "a": []string{"1", "2"}, + "b": []string{"3", "4"}, + "c": []string{"5"}, + }, got, "could not get correct merged map") +} diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 12302a229..db708f271 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -57,12 +57,12 @@ func TestHTTPExtractMultipleReuse(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/robots.txt": - w.Write([]byte(`User-agent: Googlebot + _, _ = w.Write([]byte(`User-agent: Googlebot Disallow: /a Disallow: /b Disallow: /c`)) default: - w.Write([]byte(fmt.Sprintf(`match %v`, r.URL.Path))) + _, _ = w.Write([]byte(fmt.Sprintf(`match %v`, r.URL.Path))) } })) defer ts.Close() From 393babe3c3598d9987d4a802785c856baee4eb41 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 24 Nov 2021 22:40:17 +0530 Subject: [PATCH 150/196] Made iterate of values controllable with iterate-all-values flag in http --- v2/Makefile | 4 ++++ v2/pkg/operators/operators.go | 12 ++++++++++- v2/pkg/operators/operators_test.go | 23 +++++++++++++++++++--- v2/pkg/protocols/common/generators/maps.go | 3 --- v2/pkg/protocols/http/http.go | 3 +++ v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/request_test.go | 1 + 7 files changed, 40 insertions(+), 8 deletions(-) diff --git a/v2/Makefile b/v2/Makefile index a24c1194f..cfcd3847f 100644 --- a/v2/Makefile +++ b/v2/Makefile @@ -18,5 +18,9 @@ docs: ./cmd/docgen/docgen docs.md nuclei-jsonschema.json test: $(GOTEST) -v ./... +integration: + bash ../integration_tests/run.sh +functional: + bash cmd/functional-tests/run.sh tidy: $(GOMOD) tidy \ No newline at end of file diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 6a88e0785..804965d69 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -80,8 +80,18 @@ type Result struct { // MakeDynamicValuesCallback takes an input dynamic values map and calls // the callback function with all variations of the data in input in form // of map[string]string (interface{}). -func MakeDynamicValuesCallback(input map[string][]string, callback func(map[string]interface{}) bool) { +func MakeDynamicValuesCallback(input map[string][]string, iterateAllValues bool, callback func(map[string]interface{}) bool) { output := make(map[string]interface{}, len(input)) + + if !iterateAllValues { + for k, v := range input { + if len(v) > 0 { + output[k] = v[0] + } + } + callback(output) + return + } inputIndex := make(map[string]int, len(input)) var maxValue int diff --git a/v2/pkg/operators/operators_test.go b/v2/pkg/operators/operators_test.go index 023a6c1f0..204cd57ba 100644 --- a/v2/pkg/operators/operators_test.go +++ b/v2/pkg/operators/operators_test.go @@ -15,14 +15,14 @@ func TestMakeDynamicValuesCallback(t *testing.T) { } count := 0 - MakeDynamicValuesCallback(input, func(data map[string]interface{}) bool { + MakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool { count++ require.Len(t, data, 3, "could not get correct output length") return false }) require.Equal(t, 3, count, "could not get correct result count") - t.Run("single", func(t *testing.T) { + t.Run("all", func(t *testing.T) { input := map[string][]string{ "a": []string{"1"}, "b": []string{"2"}, @@ -30,7 +30,24 @@ func TestMakeDynamicValuesCallback(t *testing.T) { } count := 0 - MakeDynamicValuesCallback(input, func(data map[string]interface{}) bool { + MakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool { + count++ + require.Len(t, data, 3, "could not get correct output length") + return false + }) + require.Equal(t, 1, count, "could not get correct result count") + }) + + t.Run("first", func(t *testing.T) { + input := map[string][]string{ + "a": []string{"1", "2"}, + "b": []string{"3"}, + "c": []string{}, + "d": []string{"A", "B", "C"}, + } + + count := 0 + MakeDynamicValuesCallback(input, false, func(data map[string]interface{}) bool { count++ require.Len(t, data, 3, "could not get correct output length") return false diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index 167dc85a9..d88ec3dbd 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -1,7 +1,6 @@ package generators import ( - "fmt" "reflect" "strings" ) @@ -39,8 +38,6 @@ func MergeMapsMany(maps ...interface{}) map[string][]string { appendToSlice(e.String(), value) } } - default: - fmt.Printf("invalid type: %v\n", v.Kind()) } } } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588d..566f4e5e4 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -176,6 +176,9 @@ type Request struct { // description: | // SkipVariablesCheck skips the check for unresolved variables in request SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` + // description: | + // IterateAllValues iterates all the values extracted from internal extractors + IterateAllValues bool `yaml:"iterate-all-values,omitempty" jsonschema:"title=iterate all values,description=Iterates all the values extracted from internal extractors"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 2c7dc0058..d1f37bafb 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -302,7 +302,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou var gotErr error var skip bool if len(gotDynamicValues) > 0 { - operators.MakeDynamicValuesCallback(gotDynamicValues, func(data map[string]interface{}) bool { + operators.MakeDynamicValuesCallback(gotDynamicValues, request.IterateAllValues, func(data map[string]interface{}) bool { if skip, gotErr = executeFunc(inputData, payloads, data); skip || gotErr != nil { return true } diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index db708f271..93016acb5 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -53,6 +53,7 @@ func TestHTTPExtractMultipleReuse(t *testing.T) { Internal: true, }}, }, + IterateAllValues: true, } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { From f6072e9a95fd22dfeaf0bd5efdb1accde072696d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 24 Nov 2021 22:44:43 +0530 Subject: [PATCH 151/196] Updating name of variable --- v2/pkg/protocols/http/http.go | 4 ++-- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/request_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 566f4e5e4..415a17baa 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -177,8 +177,8 @@ type Request struct { // SkipVariablesCheck skips the check for unresolved variables in request SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` // description: | - // IterateAllValues iterates all the values extracted from internal extractors - IterateAllValues bool `yaml:"iterate-all-values,omitempty" jsonschema:"title=iterate all values,description=Iterates all the values extracted from internal extractors"` + // IterateAll iterates all the values extracted from internal extractors + IterateAll bool `yaml:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"` } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d1f37bafb..51f029fa3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -302,7 +302,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou var gotErr error var skip bool if len(gotDynamicValues) > 0 { - operators.MakeDynamicValuesCallback(gotDynamicValues, request.IterateAllValues, func(data map[string]interface{}) bool { + operators.MakeDynamicValuesCallback(gotDynamicValues, request.IterateAll, func(data map[string]interface{}) bool { if skip, gotErr = executeFunc(inputData, payloads, data); skip || gotErr != nil { return true } diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 93016acb5..5535e84e1 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -53,7 +53,7 @@ func TestHTTPExtractMultipleReuse(t *testing.T) { Internal: true, }}, }, - IterateAllValues: true, + IterateAll: true, } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { From 7e9272776d25de1963f8a962f13cb97a391df4fb Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Wed, 24 Nov 2021 22:19:42 +0530 Subject: [PATCH 152/196] add variable support to dsl, remove dynamicValues from request struct --- v2/pkg/operators/matchers/match.go | 23 +++++++++++++++++++---- v2/pkg/protocols/http/http.go | 1 - v2/pkg/protocols/http/operators.go | 2 +- v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/network/network.go | 5 ++--- v2/pkg/protocols/network/operators.go | 2 +- v2/pkg/protocols/network/request.go | 5 ++--- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 84601be6e..d04a6149b 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,9 @@ package matchers import ( "strings" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) @@ -39,7 +42,7 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { +func (m *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) { if m.CaseInsensitive { corpus = strings.ToLower(corpus) } @@ -47,12 +50,12 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { - if dynamicValues == nil { - dynamicValues = make(map[string]interface{}) + if data == nil { + data = make(map[string]interface{}) } var err error - word, err = expressions.Evaluate(word, dynamicValues) + word, err = expressions.Evaluate(word, data) if err != nil { continue } @@ -148,6 +151,18 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { func (m *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { + if varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil { + resolvedExpression, err := expressions.Evaluate(expression.String(), data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", m.Name, err.Error()) + return false + } + expression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions()) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", m.Name, err.Error()) + return false + } + } result, err := expression.Evaluate(data) if err != nil { continue diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588d..49e7c74de 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -134,7 +134,6 @@ type Request struct { generator *generators.PayloadGenerator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client - dynamicValues map[string]interface{} // description: | // SelfContained specifies if the request is self contained. diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 1f319ccff..70fd4d4c7 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -32,7 +32,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..405e45bf4 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -231,7 +231,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou if reqURL == "" { reqURL = generatedHttpRequest.URL() } - request.dynamicValues = generatedHttpRequest.dynamicValues + // Check if hosts just keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break @@ -470,7 +470,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } - event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) if hasInteractMarkers { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 09dcfb451..385b5bb7c 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -77,9 +77,8 @@ type Request struct { generator *generators.PayloadGenerator // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions - dynamicValues map[string]interface{} + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions } type addressKV struct { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 010b11d61..d3083301b 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -23,7 +23,7 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, data)) case matchers.RegexMatcher: return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 35da81d15..484f95ae2 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -78,6 +78,7 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho } payloads := generators.BuildPayloadFromOptions(request.options.Options) + generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if request.generator != nil { iterator := request.generator.NewIterator() @@ -108,8 +109,6 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input err error ) - request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) - if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } @@ -267,7 +266,7 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input var event *output.InternalWrappedEvent if len(interactshURLs) == 0 { - event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(payloads, outputEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { wrappedEvent.OperatorsResult.PayloadValues = payloads }) callback(event) From 4bccb6cf8a830724b3bda624cb3bfe43eff31683 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 24 Nov 2021 17:09:38 +0200 Subject: [PATCH 153/196] fix: File and directory creation permission changes Directories: 0755 (5 - group and other can read cd into the directory and read it's content) Files: 0644 (4 - group and other can only read the created files) Tests files: 0777 --- v2/cmd/cve-annotate/main.go | 3 ++- v2/cmd/docgen/docgen.go | 5 +++-- v2/internal/runner/update.go | 10 +++++----- v2/internal/runner/update_test.go | 11 ++++++----- v2/pkg/catalog/config/config.go | 6 +++--- v2/pkg/protocols/file/find_test.go | 2 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/offlinehttp/find_test.go | 2 +- v2/pkg/reporting/exporters/markdown/markdown.go | 2 +- 9 files changed, 23 insertions(+), 20 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 4786ad245..45ede0420 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Ice3man543/nvd" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) @@ -135,7 +136,7 @@ func getCVEData(client *nvd.Client, filePath, data string) { } newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock) if changed { - _ = ioutil.WriteFile(filePath, []byte(newTemplate), 0777) + _ = ioutil.WriteFile(filePath, []byte(newTemplate), 0644) fmt.Printf("Wrote updated template to %s\n", filePath) } } diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index 5cd359c94..aa5405ee9 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/alecthomas/jsonschema" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) @@ -21,7 +22,7 @@ func main() { if err != nil { log.Fatalf("Could not encode docs: %s\n", err) } - err = ioutil.WriteFile(os.Args[1], data, 0777) + err = ioutil.WriteFile(os.Args[1], data, 0644) if err != nil { log.Fatalf("Could not write docs: %s\n", err) } @@ -43,7 +44,7 @@ func main() { for _, match := range pathRegex.FindAllStringSubmatch(schema, -1) { schema = strings.ReplaceAll(schema, match[0], match[1]) } - err = ioutil.WriteFile(os.Args[2], []byte(schema), 0777) + err = ioutil.WriteFile(os.Args[2], []byte(schema), 0644) if err != nil { log.Fatalf("Could not write jsonschema: %s\n", err) } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 217f34008..eaef33b39 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -54,7 +54,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju return err } configDir := filepath.Join(home, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) if err := r.readInternalConfigurationFile(home, configDir); err != nil { return errors.Wrap(err, "could not read configuration file") @@ -266,7 +266,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU } // Create the template folder if it doesn't exist - if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, os.ModePerm); err != nil { + if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { return nil, fmt.Errorf("failed to create template base folder: %s", err) } @@ -291,7 +291,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU buffer.WriteString("\n") } - if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), os.ModePerm); err != nil { + if err := ioutil.WriteFile(additionsFile, buffer.Bytes(), 0644); err != nil { return nil, errors.Wrap(err, "could not write new additions file") } return results, err @@ -331,7 +331,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat } results.totalCount++ templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath) - if err := os.MkdirAll(templateDirectory, os.ModePerm); err != nil { + if err := os.MkdirAll(templateDirectory, 0755); err != nil { return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } @@ -341,7 +341,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) { isAddition = true } - templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { templateFile.Close() return nil, fmt.Errorf("could not create uncompressed file: %s", err) diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index bc0888341..62564acd8 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -12,10 +12,11 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestDownloadReleaseAndUnzipAddition(t *testing.T) { @@ -25,7 +26,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(baseTemplates) - err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create write base file") err = zipFromDirectory("base.zip", baseTemplates) @@ -50,9 +51,9 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(newTempDir) - err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(newTempDir, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create base file") - err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(newTempDir, "new.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create new file") err = zipFromDirectory("new.zip", newTempDir) @@ -77,7 +78,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(baseTemplates) - err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), 0777) + err = ioutil.WriteFile(filepath.Join(baseTemplates, "base.yaml"), []byte("id: test"), os.ModePerm) require.Nil(t, err, "could not create write base file") err = zipFromDirectory("base.zip", baseTemplates) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 91b153cf2..87e6c672d 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -34,7 +34,7 @@ func getConfigDetails() (string, error) { return "", errors.Wrap(err, "could not get home directory") } configDir := filepath.Join(homeDir, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) return templatesConfigFile, nil } @@ -67,7 +67,7 @@ func WriteConfiguration(config *Config) error { if err != nil { return err } - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) + file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } @@ -112,7 +112,7 @@ func getIgnoreFilePath() string { home, err := os.UserHomeDir() if err == nil { configDir := filepath.Join(home, ".config", "nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) + _ = os.MkdirAll(configDir, 0755) defIgnoreFilePath = filepath.Join(configDir, nucleiIgnoreFile) return defIgnoreFilePath diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go index 7262bb26a..58eb128b6 100644 --- a/v2/pkg/protocols/file/find_test.go +++ b/v2/pkg/protocols/file/find_test.go @@ -44,7 +44,7 @@ func TestFindInputPaths(t *testing.T) { "test.js": "TEST", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } expected := []string{"config.yaml", "final.yaml", "test.js"} diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index ba057dafe..a25d5538f 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -57,7 +57,7 @@ func TestFileExecuteWithResults(t *testing.T) { "config.yaml": "TEST\r\n1.1.1.1\r\n", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } diff --git a/v2/pkg/protocols/offlinehttp/find_test.go b/v2/pkg/protocols/offlinehttp/find_test.go index d6b933ce2..367d00096 100644 --- a/v2/pkg/protocols/offlinehttp/find_test.go +++ b/v2/pkg/protocols/offlinehttp/find_test.go @@ -40,7 +40,7 @@ func TestFindResponses(t *testing.T) { "test.txt": "TEST", } for k, v := range files { - err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), 0777) + err = ioutil.WriteFile(filepath.Join(tempDir, k), []byte(v), os.ModePerm) require.Nil(t, err, "could not write temporary file") } expected := []string{"config.txt", "final.txt", "test.txt"} diff --git a/v2/pkg/reporting/exporters/markdown/markdown.go b/v2/pkg/reporting/exporters/markdown/markdown.go index 3e779ea1e..31c356af9 100644 --- a/v2/pkg/reporting/exporters/markdown/markdown.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -32,7 +32,7 @@ func New(options *Options) (*Exporter, error) { } directory = dir } - _ = os.MkdirAll(directory, os.ModePerm) + _ = os.MkdirAll(directory, 0755) return &Exporter{options: options, directory: directory}, nil } From caaa5c65947ebe9f3dbb9cf62ef7dea01b838a84 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 25 Nov 2021 19:34:01 +0530 Subject: [PATCH 154/196] Automatically generate docs for enum values --- SYNTAX-REFERENCE.md | 426 +++++++++++++----- nuclei-jsonschema.json | 4 +- v2/go.mod | 8 +- v2/go.sum | 18 + v2/pkg/model/model.go | 7 - v2/pkg/model/types/severity/severity.go | 50 ++ .../model/types/severity/severity_holder.go | 48 -- .../operators/extractors/extractor_types.go | 23 +- v2/pkg/operators/extractors/extractors.go | 7 +- v2/pkg/operators/matchers/matchers.go | 9 - v2/pkg/operators/matchers/matchers_types.go | 15 +- .../common/generators/attack_types.go | 11 +- v2/pkg/protocols/dns/dns.go | 10 - v2/pkg/protocols/dns/dns_types.go | 12 +- v2/pkg/protocols/dns/operators_test.go | 6 +- v2/pkg/protocols/dns/request_test.go | 2 +- v2/pkg/protocols/file/operators_test.go | 6 +- v2/pkg/protocols/file/request_test.go | 2 +- v2/pkg/protocols/headless/engine/action.go | 22 - .../protocols/headless/engine/action_types.go | 26 +- v2/pkg/protocols/http/http.go | 11 - v2/pkg/protocols/http/http_method_types.go | 13 +- v2/pkg/protocols/http/operators_test.go | 14 +- v2/pkg/protocols/network/network.go | 4 - .../protocols/network/network_input_types.go | 7 +- v2/pkg/protocols/network/operators_test.go | 6 +- v2/pkg/protocols/network/request_test.go | 2 +- .../protocols/offlinehttp/operators_test.go | 6 +- v2/pkg/protocols/websocket/websocket.go | 4 - v2/pkg/templates/templates_doc.go | 235 +++++++--- v2/pkg/templates/templates_doc_examples.go | 4 +- v2/pkg/templates/types/types.go | 11 +- 32 files changed, 685 insertions(+), 344 deletions(-) delete mode 100644 v2/pkg/model/types/severity/severity_holder.go diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 6c1742432..c69801fbd 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -435,19 +435,6 @@ reference: Severity of the template. - -Valid values: - - - - info - - - low - - - medium - - - high - - - critical
@@ -568,6 +555,38 @@ Appears in: +
+ +
+ + Severity + +
+
+ + + + +Enum Values: + + + - undefined + + - info + + - low + + - medium + + - high + + - critical +
+ +
+ + + ## model.Classification @@ -880,36 +899,13 @@ Valid values:
-method HTTPMethodTypeHolder +method HTTPMethodTypeHolder
Method is the HTTP Request Method. - -Valid values: - - - - GET - - - HEAD - - - POST - - - PUT - - - DELETE - - - CONNECT - - - OPTIONS - - - TRACE - - - PATCH - - - PURGE
@@ -1274,28 +1270,13 @@ Appears in:
-type MatcherTypeHolder +type MatcherTypeHolder
Type is the type of the matcher. - -Valid values: - - - - status - - - size - - - word - - - regex - - - binary - - - dsl
@@ -1608,6 +1589,50 @@ Valid values: +## MatcherTypeHolder +MatcherTypeHolder is used to hold internal type of the matcher + +Appears in: + + +- matchers.Matcher.type + + + +
+ +
+ + MatcherType + +
+
+ + + + +Enum Values: + + + - word + + - regex + + - binary + + - status + + - size + + - dsl +
+ +
+ + + + + ## extractors.Extractor Extractor is used to extract part of response using a regex. @@ -1658,24 +1683,13 @@ name: cookie-extractor
-type TypeHolder +type ExtractorTypeHolder
Type is the type of the extractor. - -Valid values: - - - - regex - - - kval - - - json - - - xpath
@@ -1911,6 +1925,46 @@ Valid values: +## ExtractorTypeHolder +ExtractorTypeHolder is used to hold internal type of the extractor + +Appears in: + + +- extractors.Extractor.type + + + +
+ +
+ + ExtractorType + +
+
+ + + + +Enum Values: + + + - regex + + - kval + + - xpath + + - json +
+ +
+ + + + + ## generators.AttackTypeHolder AttackTypeHolder is used to hold internal type of the protocol @@ -1925,6 +1979,84 @@ Appears in: +
+ +
+ + AttackType + +
+
+ + + + +Enum Values: + + + - batteringram + + - pitchfork + + - clusterbomb +
+ +
+ + + + + +## HTTPMethodTypeHolder +HTTPMethodTypeHolder is used to hold internal type of the HTTP Method + +Appears in: + + +- http.Request.method + + + +
+ +
+ + HTTPMethodType + +
+
+ + + + +Enum Values: + + + - GET + + - GET + + - POST + + - PUT + + - DELETE + + - CONNECT + + - OPTIONS + + - TRACE + + - PATCH + + - PURGE +
+ +
+ + + ## dns.Request @@ -2043,34 +2175,13 @@ name: '{{FQDN}}'
-type DNSRequestTypeHolder +type DNSRequestTypeHolder
RequestType is the type of DNS request to make. - -Valid values: - - - - A - - - NS - - - DS - - - CNAME - - - SOA - - - PTR - - - MX - - - TXT - - - AAAA
@@ -2196,6 +2307,56 @@ Resolvers to use for the dns requests +## DNSRequestTypeHolder +DNSRequestTypeHolder is used to hold internal type of the DNS type + +Appears in: + + +- dns.Request.type + + + +
+ +
+ + DNSRequestType + +
+
+ + + + +Enum Values: + + + - A + + - NS + + - DS + + - CNAME + + - SOA + + - PTR + + - MX + + - TXT + + - AAAA +
+ +
+ + + + + ## file.Request Request contains a File matching mechanism for local disk operations. @@ -2455,15 +2616,6 @@ Attack is the type of payload combinations to perform. Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. - -Valid values: - - - - batteringram - - - pitchfork - - - clusterbomb
@@ -2646,7 +2798,7 @@ data: hex_decode('50494e47')
-type NetworkInputTypeHolder +type NetworkInputTypeHolder
@@ -2722,6 +2874,42 @@ name: prefix +## NetworkInputTypeHolder +NetworkInputTypeHolder is used to hold internal type of the Network type + +Appears in: + + +- network.Input.type + + + +
+ +
+ + NetworkInputType + +
+
+ + + + +Enum Values: + + + - hex + + - text +
+ +
+ + + + + ## headless.Request Request contains a Headless protocol request to be made from a template @@ -2879,15 +3067,44 @@ Description is the optional description of the headless action
-action ActionTypeHolder +action ActionTypeHolder
Action is the type of the action to perform. +
-Valid values: +
+ + + + + +## ActionTypeHolder +ActionTypeHolder is used to hold internal type of the action + +Appears in: + + +- engine.Action.action + + + +
+ +
+ + ActionType + +
+
+ + + + +Enum Values: - navigate @@ -2931,6 +3148,8 @@ Valid values: - debug - sleep + + - waitvisible
@@ -3135,15 +3354,6 @@ Attack is the type of payload combinations to perform. Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. - -Valid values: - - - - sniper - - - pitchfork - - - clusterbomb
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index a831db3ff..aa84681b2 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -131,7 +131,7 @@ }, "type": { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/extractors.TypeHolder" + "$ref": "#/definitions/extractors.ExtractorTypeHolder" }, "regex": { "items": { @@ -194,7 +194,7 @@ "additionalProperties": false, "type": "object" }, - "extractors.TypeHolder": { + "extractors.ExtractorTypeHolder": { "enum": [ "regex", "kval", diff --git a/v2/go.mod b/v2/go.mod index 74eb7ea3f..cbed75130 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -40,7 +40,7 @@ require ( github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 - github.com/projectdiscovery/yamldoc-go v1.0.2 + github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 @@ -77,6 +77,7 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/dave/dst v0.26.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect @@ -91,6 +92,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect @@ -124,10 +126,14 @@ require ( github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/mod v0.4.2 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/tools v0.1.3 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + mvdan.cc/gofumpt v0.1.1 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index f474a7273..1f385e95e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -163,6 +163,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -641,6 +642,20 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d h1:sXbcjsLPDgOrlGXgCKbT6MMyH/hTY3OJhhwsyM2bNlI= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326 h1:/fGqkG8GlfdvlTCfvAoSA/WgEAJrmCnW5qtdd7QXnwA= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784 h1:Be2eD7oXNvCbFufVhvkiO5a0SRVN+Ri2V9pXKL2uJp8= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a h1:xGeZvil8Fe5LpGJbTGZGafnnbedNGtVcvLv3nYtmhXQ= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b h1:oSBnxdyyDU/WpNKUAXrhydgx5+JtDT7KfJR+hOEaBXk= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0 h1:mOptvTJ32yUuqQjjSfiPkPCelTWzqnts92uNOZBXZZo= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e h1:0ZxOM0Q0/ESa24L/vq3fxs9YipxfHR4Y3jM/H2ReJ5E= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -890,6 +905,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1108,6 +1124,7 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1248,6 +1265,7 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= +mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/pkg/model/model.go b/v2/pkg/model/model.go index a2a830294..f8ddb0858 100644 --- a/v2/pkg/model/model.go +++ b/v2/pkg/model/model.go @@ -50,13 +50,6 @@ type Info struct { Reference stringslice.StringSlice `json:"reference,omitempty" yaml:"reference,omitempty" jsonschema:"title=references for the template,description=Links relevant to the template"` // description: | // Severity of the template. - // - // values: - // - info - // - low - // - medium - // - high - // - critical SeverityHolder severity.Holder `json:"severity,omitempty" yaml:"severity,omitempty"` // description: | // Metadata of the template. diff --git a/v2/pkg/model/types/severity/severity.go b/v2/pkg/model/types/severity/severity.go index bb52a2eb7..5c3b28150 100644 --- a/v2/pkg/model/types/severity/severity.go +++ b/v2/pkg/model/types/severity/severity.go @@ -1,19 +1,28 @@ package severity import ( + "encoding/json" "strings" + "github.com/alecthomas/jsonschema" "github.com/pkg/errors" ) type Severity int +// name:Severity const ( + // name:undefined Undefined Severity = iota + // name:info Info + // name:low Low + // name:medium Medium + // name:high High + // name:critical Critical limit ) @@ -51,3 +60,44 @@ func normalizeValue(value string) string { func (severity Severity) String() string { return severityMappings[severity] } + +//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe +// Holder holds a Severity type. Required for un/marshalling purposes +type Holder struct { + Severity Severity `mapping:"true"` +} + +func (severityHolder Holder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "severity of the template", + Description: "Seriousness of the implications of the template", + } + for _, severity := range GetSupportedSeverities() { + gotType.Enum = append(gotType.Enum, severity.String()) + } + return gotType +} + +func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledSeverity string + if err := unmarshal(&marshalledSeverity); err != nil { + return err + } + + computedSeverity, err := toSeverity(marshalledSeverity) + if err != nil { + return err + } + + severityHolder.Severity = computedSeverity + return nil +} + +func (severityHolder *Holder) MarshalJSON() ([]byte, error) { + return json.Marshal(severityHolder.Severity.String()) +} + +func (severityHolder Holder) MarshalYAML() (interface{}, error) { + return severityHolder.Severity.String(), nil +} diff --git a/v2/pkg/model/types/severity/severity_holder.go b/v2/pkg/model/types/severity/severity_holder.go deleted file mode 100644 index ad4c2496d..000000000 --- a/v2/pkg/model/types/severity/severity_holder.go +++ /dev/null @@ -1,48 +0,0 @@ -package severity - -import ( - "encoding/json" - - "github.com/alecthomas/jsonschema" -) - -//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe -// Holder holds a Severity type. Required for un/marshalling purposes -type Holder struct { - Severity Severity -} - -func (severityHolder Holder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ - Type: "string", - Title: "severity of the template", - Description: "Seriousness of the implications of the template", - } - for _, severity := range GetSupportedSeverities() { - gotType.Enum = append(gotType.Enum, severity.String()) - } - return gotType -} - -func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error { - var marshalledSeverity string - if err := unmarshal(&marshalledSeverity); err != nil { - return err - } - - computedSeverity, err := toSeverity(marshalledSeverity) - if err != nil { - return err - } - - severityHolder.Severity = computedSeverity - return nil -} - -func (severityHolder *Holder) MarshalJSON() ([]byte, error) { - return json.Marshal(severityHolder.Severity.String()) -} - -func (severityHolder Holder) MarshalYAML() (interface{}, error) { - return severityHolder.Severity.String(), nil -} diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go index 227001a1f..23458c922 100644 --- a/v2/pkg/operators/extractors/extractor_types.go +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -11,14 +11,15 @@ import ( // ExtractorType is the type of the extractor specified type ExtractorType int +// name:ExtractorType const ( - // RegexExtractor extracts responses with regexes + // name:regex RegexExtractor ExtractorType = iota + 1 - // KValExtractor extracts responses with key:value + // name:kval KValExtractor - // XPathExtractor extracts responses with Xpath selectors + // name:xpath XPathExtractor - // JSONExtractor extracts responses with json + // name:json JSONExtractor //limit limit @@ -64,12 +65,12 @@ func (t ExtractorType) String() string { return extractorMappings[t] } -// TypeHolder is used to hold internal type of the extractor -type TypeHolder struct { - ExtractorType ExtractorType +// ExtractorTypeHolder is used to hold internal type of the extractor +type ExtractorTypeHolder struct { + ExtractorType ExtractorType `mapping:"true"` } -func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { +func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Type { gotType := &jsonschema.Type{ Type: "string", Title: "type of the extractor", @@ -81,7 +82,7 @@ func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { return gotType } -func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (holder *ExtractorTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { var marshalledTypes string if err := unmarshal(&marshalledTypes); err != nil { return err @@ -96,10 +97,10 @@ func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error return nil } -func (holder *TypeHolder) MarshalJSON() ([]byte, error) { +func (holder *ExtractorTypeHolder) MarshalJSON() ([]byte, error) { return json.Marshal(holder.ExtractorType.String()) } -func (holder TypeHolder) MarshalYAML() (interface{}, error) { +func (holder ExtractorTypeHolder) MarshalYAML() (interface{}, error) { return holder.ExtractorType.String(), nil } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index a2121efa5..693ecb092 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -16,12 +16,7 @@ type Extractor struct { Name string `yaml:"name,omitempty" jsonschema:"title=name of the extractor,description=Name of the extractor"` // description: | // Type is the type of the extractor. - // values: - // - "regex" - // - "kval" - // - "json" - // - "xpath" - Type TypeHolder `json:"name,omitempty" yaml:"type"` + Type ExtractorTypeHolder `json:"name,omitempty" yaml:"type"` // extractorType is the internal type of the extractor extractorType ExtractorType diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 3f37deb02..7929c4ea5 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -10,13 +10,6 @@ import ( type Matcher struct { // description: | // Type is the type of the matcher. - // values: - // - "status" - // - "size" - // - "word" - // - "regex" - // - "binary" - // - "dsl" Type MatcherTypeHolder `yaml:"type" jsonschema:"title=type of matcher,description=Type of the matcher,enum=status,enum=size,enum=word,enum=regex,enum=binary,enum=dsl"` // description: | // Condition is the optional condition between two matcher variables. By default, @@ -120,7 +113,6 @@ type Matcher struct { dslCompiled []*govaluate.EvaluableExpression } - // ConditionType is the type of condition for matcher type ConditionType int @@ -152,4 +144,3 @@ func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) ( } return data, matchedSnippet } - diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 336355374..ef8c51a52 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -11,18 +11,19 @@ import ( // MatcherType is the type of the matcher specified type MatcherType int +// name:MatcherType const ( - // WordsMatcher matches responses with words + // name:word WordsMatcher MatcherType = iota + 1 - // RegexMatcher matches responses with regexes + // name:regex RegexMatcher - // BinaryMatcher matches responses with words + // name:binary BinaryMatcher - // StatusMatcher matches responses with status codes + // name:status StatusMatcher - // SizeMatcher matches responses with response size + // name:size SizeMatcher - // DSLMatcher matches based upon dsl syntax + // name:dsl DSLMatcher //limit limit @@ -72,7 +73,7 @@ func (t MatcherType) String() string { // MatcherTypeHolder is used to hold internal type of the matcher type MatcherTypeHolder struct { - MatcherType MatcherType + MatcherType MatcherType `mapping:"true"` } func (t MatcherTypeHolder) String() string { diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index 769026907..735bef386 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -11,13 +11,14 @@ import ( // AttackType is the type of attack for payloads type AttackType int -// Supported values for the ProtocolType +// Supported values for the AttackType +// name:AttackType const ( - // BatteringRamAttack replaces same payload into all of the defined payload positions at once. + // name:batteringram BatteringRamAttack AttackType = iota + 1 - // PitchForkAttack replaces variables with positional value from multiple wordlists + // name:pitchfork PitchForkAttack - // ClusterbombAttack replaces variables with all possible combinations of values + // name:clusterbomb ClusterbombAttack limit ) @@ -57,7 +58,7 @@ func (t AttackType) String() string { // AttackTypeHolder is used to hold internal type of the protocol type AttackTypeHolder struct { - Value AttackType + Value AttackType `mapping:"true"` } func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 7cde7fed4..7b01d1390 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -34,16 +34,6 @@ type Request struct { Name string `yaml:"name,omitempty" jsonschema:"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for"` // description: | // RequestType is the type of DNS request to make. - // values: - // - "A" - // - "NS" - // - "DS" - // - "CNAME" - // - "SOA" - // - "PTR" - // - "MX" - // - "TXT" - // - "AAAA" RequestType DNSRequestTypeHolder `yaml:"type,omitempty" jsonschema:"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA"` // description: | // Class is the class of the DNS request. diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go index e1354fedf..9dfe42ef8 100644 --- a/v2/pkg/protocols/dns/dns_types.go +++ b/v2/pkg/protocols/dns/dns_types.go @@ -11,15 +11,25 @@ import ( // DNSRequestType is the type of the method specified type DNSRequestType int +// name:DNSRequestType const ( + // name:A A DNSRequestType = iota + 1 + // name:NS NS + // name:DS DS + // name:CNAME CNAME + // name:SOA SOA + // name:PTR PTR + // name:MX MX + // name:TXT TXT + // name:AAAA AAAA //limit limit @@ -67,7 +77,7 @@ func (t DNSRequestType) String() string { // DNSRequestTypeHolder is used to hold internal type of the DNS type type DNSRequestTypeHolder struct { - DNSRequestType DNSRequestType + DNSRequestType DNSRequestType `mapping:"true"` } func (holder DNSRequestTypeHolder) String() string { diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 5651ea422..a472aa230 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -192,7 +192,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -205,7 +205,7 @@ func TestDNSOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"rcode"}, } err = extractor.CompileExtractors() @@ -238,7 +238,7 @@ func TestDNSMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 7b6a3e6ab..46acc9937 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -35,7 +35,7 @@ func TestDNSExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index ffdddc083..37cc62d52 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -154,7 +154,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -167,7 +167,7 @@ func TestFileOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"raw"}, } err = extractor.CompileExtractors() @@ -250,7 +250,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index ba057dafe..6c15f86d0 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -37,7 +37,7 @@ func TestFileExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "raw", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 861f67a5e..e65eed8be 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -26,28 +26,6 @@ type Action struct { Description string `yaml:"description,omitempty" jsonschema:"title=description for headless action,description=Description of the headless action"` // description: | // Action is the type of the action to perform. - // values: - // - "navigate" - // - "script" - // - "click" - // - "rightclick" - // - "text" - // - "screenshot" - // - "time" - // - "select" - // - "files" - // - "waitload" - // - "getresource" - // - "extract" - // - "setmethod" - // - "addheader" - // - "setheader" - // - "deleteheader" - // - "setbody" - // - "waitevent" - // - "keyboard" - // - "debug" - // - "sleep" ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"` } diff --git a/v2/pkg/protocols/headless/engine/action_types.go b/v2/pkg/protocols/headless/engine/action_types.go index 4f71b4215..03fcf2e54 100644 --- a/v2/pkg/protocols/headless/engine/action_types.go +++ b/v2/pkg/protocols/headless/engine/action_types.go @@ -12,51 +12,73 @@ import ( type ActionType int8 // Types to be executed by the user. +// name:ActionType const ( // ActionNavigate performs a navigation to the specified URL - // URL can include nuclei payload data such as URL, Hostname, etc. + // name:navigate ActionNavigate ActionType = iota + 1 // ActionScript executes a JS snippet on the page. + // name:script ActionScript // ActionClick performs the left-click action on an Element. + // name:click ActionClick // ActionRightClick performs the right-click action on an Element. + // name:rightclick ActionRightClick // ActionTextInput performs an action for a text input + // name:text ActionTextInput // ActionScreenshot performs the screenshot action writing to a file. + // name:screenshot ActionScreenshot // ActionTimeInput performs an action on a time input. + // name:time ActionTimeInput // ActionSelectInput performs an action on a select input. + // name:select ActionSelectInput // ActionFilesInput performs an action on a file input. + // name:files ActionFilesInput // ActionWaitLoad waits for the page to stop loading. + // name:waitload ActionWaitLoad // ActionGetResource performs a get resource action on an element + // name:getresource ActionGetResource // ActionExtract performs an extraction on an element + // name:extract ActionExtract // ActionSetMethod sets the request method + // name:setmethod ActionSetMethod // ActionAddHeader adds a header to the request + // name:addheader ActionAddHeader // ActionSetHeader sets a header in the request + // name:setheader ActionSetHeader // ActionDeleteHeader deletes a header from the request + // name:deleteheader ActionDeleteHeader // ActionSetBody sets the value of the request body + // name:setbody ActionSetBody // ActionWaitEvent waits for a specific event. + // name:waitevent ActionWaitEvent // ActionKeyboard performs a keyboard action event on a page. + // name:keyboard ActionKeyboard // ActionDebug debug slows down headless and adds a sleep to each page. + // name:debug ActionDebug // ActionSleep executes a sleep for a specified duration + // name:sleep ActionSleep // ActionWaitVisible waits until an element appears. + // name:waitvisible ActionWaitVisible // limit limit @@ -143,7 +165,7 @@ func (t ActionType) String() string { // ActionTypeHolder is used to hold internal type of the action type ActionTypeHolder struct { - ActionType ActionType + ActionType ActionType `mapping:"true"` } func (holder ActionTypeHolder) String() string { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588d..fca27c162 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -55,17 +55,6 @@ type Request struct { AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. - // values: - // - "GET" - // - "HEAD" - // - "POST" - // - "PUT" - // - "DELETE" - // - "CONNECT" - // - "OPTIONS" - // - "TRACE" - // - "PATCH" - // - "PURGE" Method HTTPMethodTypeHolder `yaml:"method,omitempty" jsonschema:"title=method is the http request method,description=Method is the HTTP Request Method,enum=GET,enum=HEAD,enum=POST,enum=PUT,enum=DELETE,enum=CONNECT,enum=OPTIONS,enum=TRACE,enum=PATCH,enum=PURGE"` // description: | // Body is an optional parameter which contains HTTP Request body. diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go index 987dde979..cac0caf09 100644 --- a/v2/pkg/protocols/http/http_method_types.go +++ b/v2/pkg/protocols/http/http_method_types.go @@ -11,16 +11,27 @@ import ( // HTTPMethodType is the type of the method specified type HTTPMethodType int +// name:HTTPMethodType const ( + // name:GET HTTPGet HTTPMethodType = iota + 1 + // name:GET HTTPHead + // name:POST HTTPPost + // name:PUT HTTPPut + // name:DELETE HTTPDelete + // name:CONNECT HTTPConnect + // name:OPTIONS HTTPOptions + // name:TRACE HTTPTrace + // name:PATCH HTTPPatch + // name:PURGE HTTPPurge //limit limit @@ -69,7 +80,7 @@ func (t HTTPMethodType) String() string { // HTTPMethodTypeHolder is used to hold internal type of the HTTP Method type HTTPMethodTypeHolder struct { - MethodType HTTPMethodType + MethodType HTTPMethodType `mapping:"true"` } func (holder HTTPMethodTypeHolder) String() string { diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index ae6c644cf..689e06e79 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -166,7 +166,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -179,7 +179,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test_header"}, } err = extractor.CompileExtractors() @@ -195,7 +195,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("jq-simple", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters | .batter | .[] | .id"}, } err = extractor.CompileExtractors() @@ -207,7 +207,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-array", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".array"}, } err = extractor.CompileExtractors() @@ -219,7 +219,7 @@ func TestHTTPOperatorExtract(t *testing.T) { }) t.Run("jq-object", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor}, JSON: []string{".batters"}, } err = extractor.CompileExtractors() @@ -235,7 +235,7 @@ func TestHTTPOperatorExtract(t *testing.T) { event["body"] = exampleResponseBody extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"TEST_HEADER"}, // only applies to KVal CaseInsensitive: true, } @@ -267,7 +267,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 09dcfb451..c943b42e2 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -36,10 +36,6 @@ type Request struct { // // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. - // values: - // - "batteringram" - // - "pitchfork" - // - "clusterbomb" AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go index f96c3e492..d0c1facb3 100644 --- a/v2/pkg/protocols/network/network_input_types.go +++ b/v2/pkg/protocols/network/network_input_types.go @@ -8,11 +8,14 @@ import ( "github.com/alecthomas/jsonschema" ) -// NetworkInputType is the type of the method specified +// NetworkInputType is the type of the network input specified type NetworkInputType int +// name:NetworkInputType const ( + // name:hex hexType NetworkInputType = iota + 1 + // name:text textType //limit limit @@ -53,7 +56,7 @@ func (t NetworkInputType) String() string { // NetworkInputTypeHolder is used to hold internal type of the Network type type NetworkInputTypeHolder struct { - NetworkInputType NetworkInputType + NetworkInputType NetworkInputType `mapping:"true"` } func (holder NetworkInputTypeHolder) GetType() NetworkInputType { diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 805c0ed22..241daf4b5 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -149,7 +149,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "data", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -162,7 +162,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"request"}, } err = extractor.CompileExtractors() @@ -193,7 +193,7 @@ func TestNetworkMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }, diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 3cbff4d96..98074ba7e 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -38,7 +38,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "data", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"

.*

"}, }}, }, diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 0a3a663f0..7ee5172fb 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -139,7 +139,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ Part: "body", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } err = extractor.CompileExtractors() @@ -152,7 +152,7 @@ func TestHTTPOperatorExtract(t *testing.T) { t.Run("kval", func(t *testing.T) { extractor := &extractors.Extractor{ - Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.KValExtractor}, KVal: []string{"test-header"}, Part: "header", } @@ -184,7 +184,7 @@ func TestHTTPMakeResult(t *testing.T) { }}, Extractors: []*extractors.Extractor{{ Part: "body", - Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, + Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, }} diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index ab864f65b..d3026ee6a 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -52,10 +52,6 @@ type Request struct { // // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. - // values: - // - "sniper" - // - "pitchfork" - // - "clusterbomb" AttackType generators.AttackTypeHolder `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index eb547d0a6..b3ce14f71 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -16,14 +16,20 @@ var ( MODELClassificationDoc encoder.Doc HTTPRequestDoc encoder.Doc MATCHERSMatcherDoc encoder.Doc + MatcherTypeHolderDoc encoder.Doc EXTRACTORSExtractorDoc encoder.Doc + ExtractorTypeHolderDoc encoder.Doc GENERATORSAttackTypeHolderDoc encoder.Doc + HTTPMethodTypeHolderDoc encoder.Doc DNSRequestDoc encoder.Doc + DNSRequestTypeHolderDoc encoder.Doc FILERequestDoc encoder.Doc NETWORKRequestDoc encoder.Doc NETWORKInputDoc encoder.Doc + NetworkInputTypeHolderDoc encoder.Doc HEADLESSRequestDoc encoder.Doc ENGINEActionDoc encoder.Doc + ActionTypeHolderDoc encoder.Doc SSLRequestDoc encoder.Doc WEBSOCKETRequestDoc encoder.Doc WEBSOCKETInputDoc encoder.Doc @@ -160,13 +166,6 @@ func init() { MODELInfoDoc.Fields[5].Note = "" MODELInfoDoc.Fields[5].Description = "Severity of the template." MODELInfoDoc.Fields[5].Comments[encoder.LineComment] = "Severity of the template." - MODELInfoDoc.Fields[5].Values = []string{ - "info", - "low", - "medium", - "high", - "critical", - } MODELInfoDoc.Fields[6].Name = "metadata" MODELInfoDoc.Fields[6].Type = "map[string]string" MODELInfoDoc.Fields[6].Note = "" @@ -237,7 +236,20 @@ func init() { FieldName: "severity", }, } - SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0) + SEVERITYHolderDoc.Fields = make([]encoder.Doc, 1) + SEVERITYHolderDoc.Fields[0].Name = "" + SEVERITYHolderDoc.Fields[0].Type = "Severity" + SEVERITYHolderDoc.Fields[0].Note = "" + SEVERITYHolderDoc.Fields[0].Description = "" + SEVERITYHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + SEVERITYHolderDoc.Fields[0].EnumFields = []string{ + "undefined", + "info", + "low", + "medium", + "high", + "critical", + } MODELClassificationDoc.Type = "model.Classification" MODELClassificationDoc.Comments[encoder.LineComment] = "" @@ -348,18 +360,6 @@ func init() { HTTPRequestDoc.Fields[8].Note = "" HTTPRequestDoc.Fields[8].Description = "Method is the HTTP Request Method." HTTPRequestDoc.Fields[8].Comments[encoder.LineComment] = "Method is the HTTP Request Method." - HTTPRequestDoc.Fields[8].Values = []string{ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PURGE", - } HTTPRequestDoc.Fields[9].Name = "body" HTTPRequestDoc.Fields[9].Type = "string" HTTPRequestDoc.Fields[9].Note = "" @@ -501,14 +501,6 @@ func init() { MATCHERSMatcherDoc.Fields[0].Note = "" MATCHERSMatcherDoc.Fields[0].Description = "Type is the type of the matcher." MATCHERSMatcherDoc.Fields[0].Comments[encoder.LineComment] = "Type is the type of the matcher." - MATCHERSMatcherDoc.Fields[0].Values = []string{ - "status", - "size", - "word", - "regex", - "binary", - "dsl", - } MATCHERSMatcherDoc.Fields[1].Name = "condition" MATCHERSMatcherDoc.Fields[1].Type = "string" MATCHERSMatcherDoc.Fields[1].Note = "" @@ -607,6 +599,30 @@ func init() { "true", } + MatcherTypeHolderDoc.Type = "MatcherTypeHolder" + MatcherTypeHolderDoc.Comments[encoder.LineComment] = " MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.Description = "MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "matchers.Matcher", + FieldName: "type", + }, + } + MatcherTypeHolderDoc.Fields = make([]encoder.Doc, 1) + MatcherTypeHolderDoc.Fields[0].Name = "" + MatcherTypeHolderDoc.Fields[0].Type = "MatcherType" + MatcherTypeHolderDoc.Fields[0].Note = "" + MatcherTypeHolderDoc.Fields[0].Description = "" + MatcherTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + MatcherTypeHolderDoc.Fields[0].EnumFields = []string{ + "word", + "regex", + "binary", + "status", + "size", + "dsl", + } + EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." EXTRACTORSExtractorDoc.Description = "Extractor is used to extract part of response using a regex." @@ -649,16 +665,10 @@ func init() { EXTRACTORSExtractorDoc.Fields[0].AddExample("", "cookie-extractor") EXTRACTORSExtractorDoc.Fields[1].Name = "type" - EXTRACTORSExtractorDoc.Fields[1].Type = "TypeHolder" + EXTRACTORSExtractorDoc.Fields[1].Type = "ExtractorTypeHolder" EXTRACTORSExtractorDoc.Fields[1].Note = "" EXTRACTORSExtractorDoc.Fields[1].Description = "Type is the type of the extractor." EXTRACTORSExtractorDoc.Fields[1].Comments[encoder.LineComment] = "Type is the type of the extractor." - EXTRACTORSExtractorDoc.Fields[1].Values = []string{ - "regex", - "kval", - "json", - "xpath", - } EXTRACTORSExtractorDoc.Fields[2].Name = "regex" EXTRACTORSExtractorDoc.Fields[2].Type = "[]string" EXTRACTORSExtractorDoc.Fields[2].Note = "" @@ -727,6 +737,28 @@ func init() { "true", } + ExtractorTypeHolderDoc.Type = "ExtractorTypeHolder" + ExtractorTypeHolderDoc.Comments[encoder.LineComment] = " ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.Description = "ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "extractors.Extractor", + FieldName: "type", + }, + } + ExtractorTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ExtractorTypeHolderDoc.Fields[0].Name = "" + ExtractorTypeHolderDoc.Fields[0].Type = "ExtractorType" + ExtractorTypeHolderDoc.Fields[0].Note = "" + ExtractorTypeHolderDoc.Fields[0].Description = "" + ExtractorTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ExtractorTypeHolderDoc.Fields[0].EnumFields = []string{ + "regex", + "kval", + "xpath", + "json", + } + GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" @@ -744,7 +776,45 @@ func init() { FieldName: "attack", }, } - GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 1) + GENERATORSAttackTypeHolderDoc.Fields[0].Name = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Type = "AttackType" + GENERATORSAttackTypeHolderDoc.Fields[0].Note = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Description = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + GENERATORSAttackTypeHolderDoc.Fields[0].EnumFields = []string{ + "batteringram", + "pitchfork", + "clusterbomb", + } + + HTTPMethodTypeHolderDoc.Type = "HTTPMethodTypeHolder" + HTTPMethodTypeHolderDoc.Comments[encoder.LineComment] = " HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.Description = "HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "method", + }, + } + HTTPMethodTypeHolderDoc.Fields = make([]encoder.Doc, 1) + HTTPMethodTypeHolderDoc.Fields[0].Name = "" + HTTPMethodTypeHolderDoc.Fields[0].Type = "HTTPMethodType" + HTTPMethodTypeHolderDoc.Fields[0].Note = "" + HTTPMethodTypeHolderDoc.Fields[0].Description = "" + HTTPMethodTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + HTTPMethodTypeHolderDoc.Fields[0].EnumFields = []string{ + "GET", + "GET", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE", + } DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" @@ -794,17 +864,6 @@ func init() { DNSRequestDoc.Fields[5].Note = "" DNSRequestDoc.Fields[5].Description = "RequestType is the type of DNS request to make." DNSRequestDoc.Fields[5].Comments[encoder.LineComment] = "RequestType is the type of DNS request to make." - DNSRequestDoc.Fields[5].Values = []string{ - "A", - "NS", - "DS", - "CNAME", - "SOA", - "PTR", - "MX", - "TXT", - "AAAA", - } DNSRequestDoc.Fields[6].Name = "class" DNSRequestDoc.Fields[6].Type = "string" DNSRequestDoc.Fields[6].Note = "" @@ -848,6 +907,33 @@ func init() { DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" + DNSRequestTypeHolderDoc.Type = "DNSRequestTypeHolder" + DNSRequestTypeHolderDoc.Comments[encoder.LineComment] = " DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.Description = "DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "dns.Request", + FieldName: "type", + }, + } + DNSRequestTypeHolderDoc.Fields = make([]encoder.Doc, 1) + DNSRequestTypeHolderDoc.Fields[0].Name = "" + DNSRequestTypeHolderDoc.Fields[0].Type = "DNSRequestType" + DNSRequestTypeHolderDoc.Fields[0].Note = "" + DNSRequestTypeHolderDoc.Fields[0].Description = "" + DNSRequestTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + DNSRequestTypeHolderDoc.Fields[0].EnumFields = []string{ + "A", + "NS", + "DS", + "CNAME", + "SOA", + "PTR", + "MX", + "TXT", + "AAAA", + } + FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." FILERequestDoc.Description = "Request contains a File matching mechanism for local disk operations." @@ -940,11 +1026,6 @@ func init() { NETWORKRequestDoc.Fields[2].Note = "" NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." - NETWORKRequestDoc.Fields[2].Values = []string{ - "batteringram", - "pitchfork", - "clusterbomb", - } NETWORKRequestDoc.Fields[3].Name = "payloads" NETWORKRequestDoc.Fields[3].Type = "map[string]interface{}" NETWORKRequestDoc.Fields[3].Note = "" @@ -1032,6 +1113,26 @@ func init() { NETWORKInputDoc.Fields[3].AddExample("", "prefix") + NetworkInputTypeHolderDoc.Type = "NetworkInputTypeHolder" + NetworkInputTypeHolderDoc.Comments[encoder.LineComment] = " NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.Description = "NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "network.Input", + FieldName: "type", + }, + } + NetworkInputTypeHolderDoc.Fields = make([]encoder.Doc, 1) + NetworkInputTypeHolderDoc.Fields[0].Name = "" + NetworkInputTypeHolderDoc.Fields[0].Type = "NetworkInputType" + NetworkInputTypeHolderDoc.Fields[0].Note = "" + NetworkInputTypeHolderDoc.Fields[0].Description = "" + NetworkInputTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + NetworkInputTypeHolderDoc.Fields[0].EnumFields = []string{ + "hex", + "text", + } + HEADLESSRequestDoc.Type = "headless.Request" HEADLESSRequestDoc.Comments[encoder.LineComment] = " Request contains a Headless protocol request to be made from a template" HEADLESSRequestDoc.Description = "Request contains a Headless protocol request to be made from a template" @@ -1102,7 +1203,23 @@ func init() { ENGINEActionDoc.Fields[3].Note = "" ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." - ENGINEActionDoc.Fields[3].Values = []string{ + + ActionTypeHolderDoc.Type = "ActionTypeHolder" + ActionTypeHolderDoc.Comments[encoder.LineComment] = " ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.Description = "ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "engine.Action", + FieldName: "action", + }, + } + ActionTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ActionTypeHolderDoc.Fields[0].Name = "" + ActionTypeHolderDoc.Fields[0].Type = "ActionType" + ActionTypeHolderDoc.Fields[0].Note = "" + ActionTypeHolderDoc.Fields[0].Description = "" + ActionTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ActionTypeHolderDoc.Fields[0].EnumFields = []string{ "navigate", "script", "click", @@ -1124,6 +1241,7 @@ func init() { "keyboard", "debug", "sleep", + "waitvisible", } SSLRequestDoc.Type = "ssl.Request" @@ -1210,11 +1328,6 @@ func init() { WEBSOCKETRequestDoc.Fields[6].Note = "" WEBSOCKETRequestDoc.Fields[6].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." WEBSOCKETRequestDoc.Fields[6].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." - WEBSOCKETRequestDoc.Fields[6].Values = []string{ - "sniper", - "pitchfork", - "clusterbomb", - } WEBSOCKETRequestDoc.Fields[7].Name = "payloads" WEBSOCKETRequestDoc.Fields[7].Type = "map[string]interface{}" WEBSOCKETRequestDoc.Fields[7].Note = "" @@ -1326,14 +1439,20 @@ func GetTemplateDoc() *encoder.FileDoc { &MODELClassificationDoc, &HTTPRequestDoc, &MATCHERSMatcherDoc, + &MatcherTypeHolderDoc, &EXTRACTORSExtractorDoc, + &ExtractorTypeHolderDoc, &GENERATORSAttackTypeHolderDoc, + &HTTPMethodTypeHolderDoc, &DNSRequestDoc, + &DNSRequestTypeHolderDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, + &NetworkInputTypeHolderDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, + &ActionTypeHolderDoc, &SSLRequestDoc, &WEBSOCKETRequestDoc, &WEBSOCKETInputDoc, diff --git a/v2/pkg/templates/templates_doc_examples.go b/v2/pkg/templates/templates_doc_examples.go index 49e58eb00..bb51dd154 100644 --- a/v2/pkg/templates/templates_doc_examples.go +++ b/v2/pkg/templates/templates_doc_examples.go @@ -44,7 +44,7 @@ var ( Recursion: true, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, + {Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}}, }, }, } @@ -54,7 +54,7 @@ var ( Extensions: []string{"all"}, Operators: operators.Operators{ Extractors: []*extractors.Extractor{ - {Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, + {Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}}, }, }, } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index d26ad7f08..9551ab5b6 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -15,14 +15,23 @@ import ( type ProtocolType int // Supported values for the ProtocolType +// name:ProtocolType const ( + // name:dns DNSProtocol ProtocolType = iota + 1 + // name:file FileProtocol + // name:http HTTPProtocol + // name:headless HeadlessProtocol + // name:network NetworkProtocol + // name:workflow WorkflowProtocol + // name:ssl SSLProtocol + // name:websocket WebsocketProtocol limit InvalidProtocol @@ -69,7 +78,7 @@ func (t ProtocolType) String() string { // TypeHolder is used to hold internal type of the protocol type TypeHolder struct { - ProtocolType ProtocolType + ProtocolType ProtocolType `mapping:"true"` } func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { From f9c214a66f37a00fc29fe64232b690e6285ec59c Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 14:46:45 +0200 Subject: [PATCH 155/196] refactor: update logic refactor to make it more testable Introduced logic to test for zip slip (path traversal) --- v2/internal/runner/update.go | 111 +++++++++++++++++++----------- v2/internal/runner/update_test.go | 37 ++++++++++ 2 files changed, 109 insertions(+), 39 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index eaef33b39..55aec7049 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -316,60 +316,42 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat // If the path isn't found in new update after being read from the previous checksum, // it is removed. This allows us fine-grained control over the download process // as well as solves a long problem with nuclei-template updates. - checksumFile := filepath.Join(r.templatesConfig.TemplatesDirectory, ".checksum") + configuredTemplateDirectory := r.templatesConfig.TemplatesDirectory + checksumFile := filepath.Join(configuredTemplateDirectory, ".checksum") templateChecksumsMap, _ := createTemplateChecksumsMap(checksumFile) for _, zipTemplateFile := range zipReader.File { - directory, name := filepath.Split(zipTemplateFile.Name) - if name == "" { + templateAbsolutePath, skipFile, err := calculateTemplateAbsolutePath(zipTemplateFile.Name, configuredTemplateDirectory) + if err != nil { + return nil, err + } + if skipFile { continue } - paths := strings.Split(directory, string(os.PathSeparator)) - finalPath := filepath.Join(paths[1:]...) - - if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { - continue - } - results.totalCount++ - templateDirectory := filepath.Join(r.templatesConfig.TemplatesDirectory, finalPath) - if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) - } - - templatePath := filepath.Join(templateDirectory, name) isAddition := false - if _, statErr := os.Stat(templatePath); os.IsNotExist(statErr) { + if _, statErr := os.Stat(templateAbsolutePath); os.IsNotExist(statErr) { isAddition = true } - templateFile, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + + newTemplateChecksum, err := writeUnZippedTemplateFile(err, templateAbsolutePath, zipTemplateFile) if err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not create uncompressed file: %s", err) + return nil, err } - zipTemplateFileReader, err := zipTemplateFile.Open() + oldTemplateChecksum, checksumOk := templateChecksumsMap[templateAbsolutePath] + + relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) if err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not open archive to extract file: %s", err) + return nil, fmt.Errorf("could not calculate relative path for template: %s. %s", templateAbsolutePath, err) } - hasher := md5.New() - // Save file and also read into hasher for md5 - if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, hasher)); err != nil { - templateFile.Close() - return nil, fmt.Errorf("could not write template file: %s", err) - } - templateFile.Close() - - oldChecksum, checksumOK := templateChecksumsMap[templatePath] - - checksum := hex.EncodeToString(hasher.Sum(nil)) if isAddition { - results.additions = append(results.additions, filepath.Join(finalPath, name)) - } else if checksumOK && oldChecksum[0] != checksum { - results.modifications = append(results.modifications, filepath.Join(finalPath, name)) + results.additions = append(results.additions, relativeTemplatePath) + } else if checksumOk && oldTemplateChecksum[0] != newTemplateChecksum { + results.modifications = append(results.modifications, relativeTemplatePath) } - results.checksums[templatePath] = checksum + results.checksums[templateAbsolutePath] = newTemplateChecksum + results.totalCount++ } // If we don't find the previous file in the newly downloaded list, @@ -378,12 +360,63 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat _, ok := results.checksums[templatePath] if !ok && templateChecksums[0] == templateChecksums[1] { _ = os.Remove(templatePath) - results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) + results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, configuredTemplateDirectory), string(os.PathSeparator))) } } return results, nil } +func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { + templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return "", fmt.Errorf("could not create template file: %s", err) + } + + zipTemplateFileReader, err := zipTemplateFile.Open() + if err != nil { + _ = templateFile.Close() + return "", fmt.Errorf("could not open archive to extract file: %s", err) + } + + md5Hash := md5.New() + + // Save file and also read into hash.Hash for md5 + if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { + _ = templateFile.Close() + return "", fmt.Errorf("could not write template file: %s", err) + } + + if err := templateFile.Close(); err != nil { + return "", fmt.Errorf("could not close file newly created template file: %s", err) + } + + checksum := hex.EncodeToString(md5Hash.Sum(nil)) + return checksum, nil +} + +func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) { + directory, fileName := filepath.Split(zipFilePath) + + if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") { + return "", true, nil + } + + directoryPathChunks := strings.Split(directory, string(os.PathSeparator)) + relativeDirectoryPathWithoutZipRoot := filepath.Join(directoryPathChunks[1:]...) + + if strings.HasPrefix(relativeDirectoryPathWithoutZipRoot, ".") { + return "", true, nil + } + + templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) + + if err := os.MkdirAll(templateDirectory, 0755); err != nil { + return "", false, fmt.Errorf("failed to create template folder: %s. %s", templateDirectory, err) + } + + return filepath.Join(templateDirectory, fileName), false, nil +} + // createTemplateChecksumsMap reads the previous checksum file from the disk. // Creates a map of template paths and their previous and currently calculated checksums as values. func createTemplateChecksumsMap(checksumsFilePath string) (map[string][2]string, error) { diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index 62564acd8..e8153b42e 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -119,6 +119,43 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") } +func TestCalculateTemplateAbsolutePath(t *testing.T) { + configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") + defer os.RemoveAll(configuredTemplateDirectory) + + t.Run("positive scenarios", func(t *testing.T) { + zipFilePathsExpectedPathsMap := map[string]string{ + "nuclei-templates/cve/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test.yaml"), + "nuclei-templates/cve/test/test.yaml": filepath.Join(configuredTemplateDirectory, "cve/test/test.yaml"), + } + + for filePathFromZip, expectedTemplateAbsPath := range zipFilePathsExpectedPathsMap { + calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) + require.Nil(t, err) + require.Equal(t, expectedTemplateAbsPath, calculatedTemplateAbsPath) + require.False(t, skipFile) + } + }) + + t.Run("negative scenarios", func(t *testing.T) { + filePathsFromZip := []string{ + "./../nuclei-templates/../cve/test.yaml", + "nuclei-templates/../cve/test.yaml", + "nuclei-templates/cve/../test.yaml", + "nuclei-templates/././../cve/test.yaml", + "nuclei-templates/.././../cve/test.yaml", + "nuclei-templates/.././../cve/../test.yaml", + } + + for _, filePathFromZip := range filePathsFromZip { + calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) + require.Nil(t, err) + require.True(t, skipFile) + require.Equal(t, "", calculatedTemplateAbsPath) + } + }) +} + func zipFromDirectory(zipPath, directory string) error { file, err := os.Create(zipPath) if err != nil { From fdd22ab668e18f001cbb56ce721e80b225b84667 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 15:18:46 +0200 Subject: [PATCH 156/196] refactor: Wrap errors using %w in fmt.Errorf see: * https://github.com/xxpxxxxp/intellij-plugin-golangci-lint/blob/master/explanation/goerr113.md * https://go.dev/blog/go1.13-errors#wrapping-errors-with-w --- v2/internal/runner/proxy.go | 2 +- v2/internal/runner/update.go | 24 ++++++++++++------------ v2/pkg/protocols/http/build_request.go | 4 ++-- v2/pkg/protocols/http/raw/raw.go | 8 ++++---- v2/pkg/reporting/trackers/jira/jira.go | 5 +++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index e254472a7..3d2e47fd8 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -28,7 +28,7 @@ func loadProxyServers(options *types.Options) error { } else if fileutil.FileExists(p) { file, err := os.Open(p) if err != nil { - return fmt.Errorf("could not open proxy file: %s", err) + return fmt.Errorf("could not open proxy file: %w", err) } defer file.Close() scanner := bufio.NewScanner(file) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 55aec7049..8a0fa6c5a 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -242,12 +242,12 @@ func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.Repositor func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) (*templateUpdateResults, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { - return nil, fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to create HTTP request to %s: %w", downloadURL, err) } res, err := http.DefaultClient.Do(req) if err != nil { - return nil, fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed to download a release file from %s: %w", downloadURL, err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { @@ -256,23 +256,23 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU buf, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("failed to create buffer for zip file: %s", err) + return nil, fmt.Errorf("failed to create buffer for zip file: %w", err) } reader := bytes.NewReader(buf) zipReader, err := zip.NewReader(reader, reader.Size()) if err != nil { - return nil, fmt.Errorf("failed to uncompress zip file: %s", err) + return nil, fmt.Errorf("failed to uncompress zip file: %w", err) } // Create the template folder if it doesn't exist if err := os.MkdirAll(r.templatesConfig.TemplatesDirectory, 0755); err != nil { - return nil, fmt.Errorf("failed to create template base folder: %s", err) + return nil, fmt.Errorf("failed to create template base folder: %w", err) } results, err := r.compareAndWriteTemplates(zipReader) if err != nil { - return nil, fmt.Errorf("failed to write templates: %s", err) + return nil, fmt.Errorf("failed to write templates: %w", err) } if r.options.Verbose { @@ -342,7 +342,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat relativeTemplatePath, err := filepath.Rel(configuredTemplateDirectory, templateAbsolutePath) if err != nil { - return nil, fmt.Errorf("could not calculate relative path for template: %s. %s", templateAbsolutePath, err) + return nil, fmt.Errorf("could not calculate relative path for template: %s. %w", templateAbsolutePath, err) } if isAddition { @@ -369,13 +369,13 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - return "", fmt.Errorf("could not create template file: %s", err) + return "", fmt.Errorf("could not create template file: %w", err) } zipTemplateFileReader, err := zipTemplateFile.Open() if err != nil { _ = templateFile.Close() - return "", fmt.Errorf("could not open archive to extract file: %s", err) + return "", fmt.Errorf("could not open archive to extract file: %w", err) } md5Hash := md5.New() @@ -383,11 +383,11 @@ func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTempla // Save file and also read into hash.Hash for md5 if _, err := io.Copy(templateFile, io.TeeReader(zipTemplateFileReader, md5Hash)); err != nil { _ = templateFile.Close() - return "", fmt.Errorf("could not write template file: %s", err) + return "", fmt.Errorf("could not write template file: %w", err) } if err := templateFile.Close(); err != nil { - return "", fmt.Errorf("could not close file newly created template file: %s", err) + return "", fmt.Errorf("could not close file newly created template file: %w", err) } checksum := hex.EncodeToString(md5Hash.Sum(nil)) @@ -411,7 +411,7 @@ func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory stri templateDirectory := filepath.Join(configuredTemplateDirectory, relativeDirectoryPathWithoutZipRoot) if err := os.MkdirAll(templateDirectory, 0755); err != nil { - return "", false, fmt.Errorf("failed to create template folder: %s. %s", templateDirectory, err) + return "", false, fmt.Errorf("failed to create template folder: %s. %w", templateDirectory, err) } return filepath.Join(templateDirectory, fileName), false, nil diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc8..0fb0a1f0f 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -124,7 +124,7 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int reader := bufio.NewReader(strings.NewReader(data)) s, err := reader.ReadString('\n') if err != nil { - return nil, fmt.Errorf("could not read request: %s", err) + return nil, fmt.Errorf("could not read request: %w", err) } parts := strings.Split(s, " ") @@ -133,7 +133,7 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int } parsed, err := url.Parse(parts[1]) if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) + return nil, fmt.Errorf("could not parse request URL: %w", err) } values := generators.MergeMaps( generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index e2d7f8ebe..df4ff5b65 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -33,7 +33,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { parsedURL, err := url.Parse(baseURL) if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) + return nil, fmt.Errorf("could not parse request URL: %w", err) } if unsafe { @@ -42,7 +42,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { reader := bufio.NewReader(strings.NewReader(request)) s, err := reader.ReadString('\n') if err != nil { - return nil, fmt.Errorf("could not read request: %s", err) + return nil, fmt.Errorf("could not read request: %w", err) } parts := strings.Split(s, " ") @@ -101,7 +101,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if !unsafe && strings.HasPrefix(parts[1], "http") { parsed, parseErr := url.Parse(parts[1]) if parseErr != nil { - return nil, fmt.Errorf("could not parse request URL: %s", parseErr) + return nil, fmt.Errorf("could not parse request URL: %w", parseErr) } rawRequest.Path = parsed.Path @@ -133,7 +133,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { // Set the request body b, err := ioutil.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("could not read request body: %s", err) + return nil, fmt.Errorf("could not read request body: %w", err) } rawRequest.Data = string(b) if !mutlipartRequest { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 16ef1b2a7..dede2b2ed 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/andygrunwald/go-jira" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -103,7 +104,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { d, _ := ioutil.ReadAll(resp.Body) data = string(d) } - return fmt.Errorf("%s => %s", err, data) + return fmt.Errorf("%w => %s", err, data) } return nil } @@ -140,7 +141,7 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, erro d, _ := ioutil.ReadAll(resp.Body) data = string(d) } - return "", fmt.Errorf("%s => %s", err, data) + return "", fmt.Errorf("%w => %s", err, data) } switch resp.Total { From bebe5cbcfd06d9429628c6bee170bb455f3ed574 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 15:39:10 +0200 Subject: [PATCH 157/196] refactor: Remove redundant character escapes from regexes --- v2/cmd/docgen/docgen.go | 2 +- v2/pkg/protocols/common/expressions/expressions.go | 3 ++- v2/pkg/protocols/common/expressions/variables.go | 2 +- v2/pkg/protocols/offlinehttp/read_response.go | 2 +- v2/pkg/templates/preprocessors.go | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index aa5405ee9..1c8e9779a 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z\.]+)`) +var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) func main() { // Generate yaml syntax documentation diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 5a4ba2f3a..4f3a4b613 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -4,12 +4,13 @@ import ( "regexp" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) -var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) +var templateExpressionRegex = regexp.MustCompile(`(?m){{[^}]+}}["')}]*`) // Evaluate checks if the match contains a dynamic variable, for each // found one we will check if it's an expression and can diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 26b566317..2aba5c324 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -6,7 +6,7 @@ import ( "strings" ) -var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) +var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|{){2}([^}]+)(?:%7[D|d]|}){2}["')}]*`) // ContainsUnresolvedVariables returns an error with variable names if the passed // input contains unresolved {{}} variables. diff --git a/v2/pkg/protocols/offlinehttp/read_response.go b/v2/pkg/protocols/offlinehttp/read_response.go index f864b5ab8..c6e1d22d1 100644 --- a/v2/pkg/protocols/offlinehttp/read_response.go +++ b/v2/pkg/protocols/offlinehttp/read_response.go @@ -8,7 +8,7 @@ import ( "strings" ) -var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `) +var noMinor = regexp.MustCompile(`HTTP/([0-9]) `) // readResponseFromString reads a raw http response from a string. func readResponseFromString(data string) (*http.Response, error) { diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index ef6edeb1c..6a1355e26 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -12,7 +12,7 @@ type Preprocessor interface { Process(data []byte) []byte } -var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`) +var preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) // expandPreprocessors expands the pre-processors if any for a template data. func (t *Template) expandPreprocessors(data []byte) []byte { From db0d2b00394ddb48347783f3a3132f373e73327d Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 16:33:16 +0200 Subject: [PATCH 158/196] fix: markdown anchor corrections --- DESIGN.md | 4 ++-- README_CN.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 807852739..8ff4dcb92 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -554,7 +554,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/reporting](./v2/pkg/reporting) - Reporting modules for nuclei. - [v2/pkg/reporting/exporters/sarif](./v2/pkg/reporting/exporters/sarif) - Sarif Result Exporter - [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter -- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/e) - Elasticsearch Result Exporter +- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/es) - Elasticsearch Result Exporter - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results - [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter @@ -577,7 +577,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/model](./v2/pkg/model) - Template Info + misc - [v2/pkg/templates](./v2/pkg/templates) - Templates core starting point - [v2/pkg/templates/cache](./v2/pkg/templates/cache) - Templates cache -- [v2/pkg/protocols](./v2/pkg/protocol) - Protocol Specification +- [v2/pkg/protocols](./v2/pkg/protocols) - Protocol Specification - [v2/pkg/protocols/file](./v2/pkg/protocols/file) - File protocol - [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol - [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support diff --git a/README_CN.md b/README_CN.md index 0c4aa69c0..e79f28743 100644 --- a/README_CN.md +++ b/README_CN.md @@ -29,7 +29,7 @@ Nuclei是一个基于模板的、可配置攻击目标的扫描快速工具， - [安装](#安装) - [Nuclei模板](#nuclei模板) - [用法](#用法) -- [运行Nuclei](#运行nuclei) +- [运行Nuclei](#运行Nuclei) - [排除模板](#排除模板) - [致谢](#致谢) From c8d009654ce0b5f4b1d22c3aa472e42f0e5b6147 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 16:57:43 +0200 Subject: [PATCH 159/196] fix: Receiver mismatch warnings --- v2/pkg/operators/matchers/compile.go | 49 ++++++++++---------- v2/pkg/operators/matchers/match.go | 50 ++++++++++----------- v2/pkg/operators/matchers/matchers.go | 10 ++--- v2/pkg/operators/matchers/matchers_types.go | 4 +- v2/pkg/templates/preprocessors.go | 2 +- v2/pkg/templates/templates.go | 18 ++++---- 6 files changed, 65 insertions(+), 68 deletions(-) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 79d062a7c..b571085c4 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -12,74 +12,73 @@ import ( ) // CompileMatchers performs the initial setup operation on a matcher -func (m *Matcher) CompileMatchers() error { +func (matcher *Matcher) CompileMatchers() error { var ok bool // Support hexadecimal encoding for matchers too. - if m.Encoding == "hex" { - for i, word := range m.Words { + if matcher.Encoding == "hex" { + for i, word := range matcher.Words { if decoded, err := hex.DecodeString(word); err == nil && len(decoded) > 0 { - m.Words[i] = string(decoded) + matcher.Words[i] = string(decoded) } } } // Set up the matcher type - computedType, err := toMatcherTypes(m.GetType().String()) + computedType, err := toMatcherTypes(matcher.GetType().String()) if err != nil { - return fmt.Errorf("unknown matcher type specified: %s", m.Type) + return fmt.Errorf("unknown matcher type specified: %s", matcher.Type) } - m.matcherType = computedType + matcher.matcherType = computedType // By default, match on body if user hasn't provided any specific items - if m.Part == "" { - m.Part = "body" + if matcher.Part == "" { + matcher.Part = "body" } - // Compile the regexes - for _, regex := range m.Regex { + for _, regex := range matcher.Regex { compiled, err := regexp.Compile(regex) if err != nil { return fmt.Errorf("could not compile regex: %s", regex) } - m.regexCompiled = append(m.regexCompiled, compiled) + matcher.regexCompiled = append(matcher.regexCompiled, compiled) } // Compile and validate binary Values in matcher - for _, value := range m.Binary { + for _, value := range matcher.Binary { if decoded, err := hex.DecodeString(value); err != nil { return fmt.Errorf("could not hex decode binary: %s", value) } else { - m.binaryDecoded = append(m.binaryDecoded, string(decoded)) + matcher.binaryDecoded = append(matcher.binaryDecoded, string(decoded)) } } // Compile the dsl expressions - for _, expr := range m.DSL { + for _, expr := range matcher.DSL { compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) if err != nil { return fmt.Errorf("could not compile dsl: %s", expr) } - m.dslCompiled = append(m.dslCompiled, compiled) + matcher.dslCompiled = append(matcher.dslCompiled, compiled) } // Set up the condition type, if any. - if m.Condition != "" { - m.condition, ok = ConditionTypes[m.Condition] + if matcher.Condition != "" { + matcher.condition, ok = ConditionTypes[matcher.Condition] if !ok { - return fmt.Errorf("unknown condition specified: %s", m.Condition) + return fmt.Errorf("unknown condition specified: %s", matcher.Condition) } } else { - m.condition = ORCondition + matcher.condition = ORCondition } - if m.CaseInsensitive { - if m.GetType() != WordsMatcher { - return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", m.Type) + if matcher.CaseInsensitive { + if matcher.GetType() != WordsMatcher { + return fmt.Errorf("case-insensitive flag is supported only for 'word' matchers (not '%s')", matcher.Type) } - for i := range m.Words { - m.Words[i] = strings.ToLower(m.Words[i]) + for i := range matcher.Words { + matcher.Words[i] = strings.ToLower(matcher.Words[i]) } } return nil diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 84601be6e..cda09a7a8 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -7,11 +7,11 @@ import ( ) // MatchStatusCode matches a status code check against a corpus -func (m *Matcher) MatchStatusCode(statusCode int) bool { +func (matcher *Matcher) MatchStatusCode(statusCode int) bool { // Iterate over all the status codes accepted as valid // // Status codes don't support AND conditions. - for _, status := range m.Status { + for _, status := range matcher.Status { // Continue if the status codes don't match if statusCode != status { continue @@ -23,11 +23,11 @@ func (m *Matcher) MatchStatusCode(statusCode int) bool { } // MatchSize matches a size check against a corpus -func (m *Matcher) MatchSize(length int) bool { +func (matcher *Matcher) MatchSize(length int) bool { // Iterate over all the sizes accepted as valid // // Sizes codes don't support AND conditions. - for _, size := range m.Size { + for _, size := range matcher.Size { // Continue if the size doesn't match if length != size { continue @@ -39,14 +39,14 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { - if m.CaseInsensitive { +func (matcher *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + if matcher.CaseInsensitive { corpus = strings.ToLower(corpus) } var matchedWords []string // Iterate over all the words accepted as valid - for i, word := range m.Words { + for i, word := range matcher.Words { if dynamicValues == nil { dynamicValues = make(map[string]interface{}) } @@ -60,7 +60,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} if !strings.Contains(corpus, word) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -68,14 +68,14 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, []string{word} } matchedWords = append(matchedWords, word) // If we are at the end of the words, return with true - if len(m.Words)-1 == i { + if len(matcher.Words)-1 == i { return true, matchedWords } } @@ -83,15 +83,15 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) (bool, []string) { +func (matcher *Matcher) MatchRegex(corpus string) (bool, []string) { var matchedRegexes []string // Iterate over all the regexes accepted as valid - for i, regex := range m.regexCompiled { + for i, regex := range matcher.regexCompiled { // Continue if the regex doesn't match if !regex.MatchString(corpus) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -100,14 +100,14 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { currentMatches := regex.FindAllString(corpus, -1) // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, currentMatches } matchedRegexes = append(matchedRegexes, currentMatches...) // If we are at the end of the regex, return with true - if len(m.regexCompiled)-1 == i { + if len(matcher.regexCompiled)-1 == i { return true, matchedRegexes } } @@ -115,14 +115,14 @@ func (m *Matcher) MatchRegex(corpus string) (bool, []string) { } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) (bool, []string) { +func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { var matchedBinary []string // Iterate over all the words accepted as valid - for i, binary := range m.binaryDecoded { + for i, binary := range matcher.binaryDecoded { if !strings.Contains(corpus, binary) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false, []string{} } // Continue with the flow since it's an OR Condition. @@ -130,14 +130,14 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true, []string{binary} } matchedBinary = append(matchedBinary, binary) // If we are at the end of the words, return with true - if len(m.Binary)-1 == i { + if len(matcher.Binary)-1 == i { return true, matchedBinary } } @@ -145,9 +145,9 @@ func (m *Matcher) MatchBinary(corpus string) (bool, []string) { } // MatchDSL matches on a generic map result -func (m *Matcher) MatchDSL(data map[string]interface{}) bool { +func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid - for i, expression := range m.dslCompiled { + for i, expression := range matcher.dslCompiled { result, err := expression.Evaluate(data) if err != nil { continue @@ -160,7 +160,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { if !ok || !bResult { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. - if m.condition == ANDCondition { + if matcher.condition == ANDCondition { return false } // Continue with the flow since it's an OR Condition. @@ -168,12 +168,12 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { } // If the condition was an OR, return on the first match. - if m.condition == ORCondition { + if matcher.condition == ORCondition { return true } // If we are at the end of the dsl, return with true - if len(m.dslCompiled)-1 == i { + if len(matcher.dslCompiled)-1 == i { return true } } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 3f37deb02..c0dbd186b 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -120,7 +120,6 @@ type Matcher struct { dslCompiled []*govaluate.EvaluableExpression } - // ConditionType is the type of condition for matcher type ConditionType int @@ -138,18 +137,17 @@ var ConditionTypes = map[string]ConditionType{ } // Result reverts the results of the match if the matcher is of type negative. -func (m *Matcher) Result(data bool) bool { - if m.Negative { +func (matcher *Matcher) Result(data bool) bool { + if matcher.Negative { return !data } return data } // ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string -func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { - if m.Negative { +func (matcher *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if matcher.Negative { return !data, []string{} } return data, matchedSnippet } - diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 336355374..143cfee01 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -39,8 +39,8 @@ var MatcherTypes = map[MatcherType]string{ } //GetType returns the type of the matcher -func (e *Matcher) GetType() MatcherType { - return e.Type.MatcherType +func (matcher *Matcher) GetType() MatcherType { + return matcher.Type.MatcherType } // GetSupportedMatcherTypes returns list of supported types diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index 6a1355e26..cc76ddc7d 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -15,7 +15,7 @@ type Preprocessor interface { var preprocessorRegex = regexp.MustCompile(`{{([a-z0-9_]+)}}`) // expandPreprocessors expands the pre-processors if any for a template data. -func (t *Template) expandPreprocessors(data []byte) []byte { +func (template *Template) expandPreprocessors(data []byte) []byte { foundMap := make(map[string]struct{}) for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d34cd1ae1..a7f272732 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -96,23 +96,23 @@ var TemplateProtocols = []string{ } // Type returns the type of the template -func (t *Template) Type() types.ProtocolType { +func (template *Template) Type() types.ProtocolType { switch { - case len(t.RequestsDNS) > 0: + case len(template.RequestsDNS) > 0: return types.DNSProtocol - case len(t.RequestsFile) > 0: + case len(template.RequestsFile) > 0: return types.FileProtocol - case len(t.RequestsHTTP) > 0: + case len(template.RequestsHTTP) > 0: return types.HTTPProtocol - case len(t.RequestsHeadless) > 0: + case len(template.RequestsHeadless) > 0: return types.HeadlessProtocol - case len(t.RequestsNetwork) > 0: + case len(template.RequestsNetwork) > 0: return types.NetworkProtocol - case len(t.Workflow.Workflows) > 0: + case len(template.Workflow.Workflows) > 0: return types.WorkflowProtocol - case len(t.RequestsSSL) > 0: + case len(template.RequestsSSL) > 0: return types.SSLProtocol - case len(t.RequestsWebsocket) > 0: + case len(template.RequestsWebsocket) > 0: return types.WebsocketProtocol default: return types.InvalidProtocol From 3fd1f57b96336abc103e8532b5a8854d3fe3660e Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:03:56 +0200 Subject: [PATCH 160/196] refactor: godoc and comment uniformization Adding space after // and before the godoc/comment --- v2/cmd/integration-test/http.go | 2 +- v2/internal/runner/options.go | 3 ++- v2/internal/runner/proxy.go | 2 +- v2/pkg/operators/extractors/extractor_types.go | 1 - v2/pkg/operators/matchers/matchers_types.go | 1 - v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go | 5 +++-- v2/pkg/protocols/dns/dns_types.go | 1 - v2/pkg/protocols/http/http_method_types.go | 1 - v2/pkg/protocols/network/network_input_types.go | 1 - v2/pkg/testutils/testutils.go | 3 ++- 10 files changed, 9 insertions(+), 11 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 0b2bb247a..1d653afa2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -40,7 +40,7 @@ var httpTestcases = map[string]testutils.TestCase{ type httpInteractshRequest struct{} -// Executes executes a test case and returns an error if occurred +// Execute executes a test case and returns an error if occurred func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..317bd59bb 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/go-playground/validator/v10" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -93,7 +94,7 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - //loading the proxy server list from file or cli and test the connectivity + // loading the proxy server list from file or cli and test the connectivity if err := loadProxyServers(options); err != nil { return err } diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go index 3d2e47fd8..6aca4be6a 100644 --- a/v2/internal/runner/proxy.go +++ b/v2/internal/runner/proxy.go @@ -117,7 +117,7 @@ func validateProxyURL(proxy string) (url.URL, error) { return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") } -//isSupportedProtocol checks given protocols are supported +// isSupportedProtocol checks given protocols are supported func isSupportedProtocol(value string) bool { return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 } diff --git a/v2/pkg/operators/extractors/extractor_types.go b/v2/pkg/operators/extractors/extractor_types.go index 227001a1f..330cbd094 100644 --- a/v2/pkg/operators/extractors/extractor_types.go +++ b/v2/pkg/operators/extractors/extractor_types.go @@ -20,7 +20,6 @@ const ( XPathExtractor // JSONExtractor extracts responses with json JSONExtractor - //limit limit ) diff --git a/v2/pkg/operators/matchers/matchers_types.go b/v2/pkg/operators/matchers/matchers_types.go index 143cfee01..adeb0b130 100644 --- a/v2/pkg/operators/matchers/matchers_types.go +++ b/v2/pkg/operators/matchers/matchers_types.go @@ -24,7 +24,6 @@ const ( SizeMatcher // DSLMatcher matches based upon dsl syntax DSLMatcher - //limit limit ) diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 14aefff8f..0abf1b524 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/bluele/gcache" + "github.com/projectdiscovery/gologger" ) @@ -16,7 +17,7 @@ import ( // It uses an LRU cache internally for skipping unresponsive hosts // that remain so for a duration. type Cache struct { - MaxHostError int + MaxHostError int verbose bool failedTargets gcache.Cache } @@ -64,7 +65,7 @@ func (c *Cache) normalizeCacheValue(value string) string { } // ErrUnresponsiveHost is returned when a host is unresponsive -//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") +// var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") // Check returns true if a host should be skipped as it has been // unresponsive for a certain number of times. diff --git a/v2/pkg/protocols/dns/dns_types.go b/v2/pkg/protocols/dns/dns_types.go index e1354fedf..a839862ad 100644 --- a/v2/pkg/protocols/dns/dns_types.go +++ b/v2/pkg/protocols/dns/dns_types.go @@ -21,7 +21,6 @@ const ( MX TXT AAAA - //limit limit ) diff --git a/v2/pkg/protocols/http/http_method_types.go b/v2/pkg/protocols/http/http_method_types.go index 987dde979..f68d9ac89 100644 --- a/v2/pkg/protocols/http/http_method_types.go +++ b/v2/pkg/protocols/http/http_method_types.go @@ -22,7 +22,6 @@ const ( HTTPTrace HTTPPatch HTTPPurge - //limit limit ) diff --git a/v2/pkg/protocols/network/network_input_types.go b/v2/pkg/protocols/network/network_input_types.go index f96c3e492..88b07e42e 100644 --- a/v2/pkg/protocols/network/network_input_types.go +++ b/v2/pkg/protocols/network/network_input_types.go @@ -14,7 +14,6 @@ type NetworkInputType int const ( hexType NetworkInputType = iota + 1 textType - //limit limit ) diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 982fbabc3..51fec82fd 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -4,6 +4,7 @@ import ( "go.uber.org/ratelimit" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -131,7 +132,7 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro } } -// Write writes the event to file and/or screen. +// WriteFailure writes the event to file and/or screen. func (m *MockOutputWriter) WriteFailure(result output.InternalEvent) error { return nil } From 47340f06b09df6db14f8ab8d71a420df6e00655a Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:09:20 +0200 Subject: [PATCH 161/196] refactor: uniformly sorted imports --- DESIGN.md | 3 ++- v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/loader.go | 1 + v2/cmd/integration-test/websocket.go | 1 + v2/pkg/catalog/find.go | 1 + v2/pkg/core/execute.go | 5 +++-- v2/pkg/core/inputs/hybrid/hmap.go | 1 + v2/pkg/core/workflow_execute.go | 5 +++-- v2/pkg/core/workpool.go | 3 ++- v2/pkg/operators/common/dsl/dsl.go | 3 ++- v2/pkg/operators/common/dsl/dsl_test.go | 3 ++- v2/pkg/operators/extractors/extract.go | 3 +-- v2/pkg/parsers/parser_test.go | 3 ++- v2/pkg/protocols/common/generators/generators.go | 1 + v2/pkg/protocols/common/generators/generators_test.go | 3 ++- v2/pkg/protocols/common/protocolinit/init.go | 1 + v2/pkg/protocols/common/protocolstate/state.go | 1 + v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/headless/engine/http_client.go | 3 ++- v2/pkg/protocols/http/build_request_test.go | 3 ++- v2/pkg/protocols/http/request_generator_test.go | 3 ++- v2/pkg/protocols/http/utils.go | 5 +++-- v2/pkg/protocols/protocols.go | 1 + v2/pkg/protocols/ssl/ssl.go | 1 + v2/pkg/protocols/ssl/ssl_test.go | 3 ++- v2/pkg/protocols/websocket/websocket.go | 1 + v2/pkg/reporting/dedupe/dedupe_test.go | 3 ++- v2/pkg/reporting/exporters/es/elasticsearch.go | 6 +++--- v2/pkg/reporting/reporting.go | 2 +- v2/pkg/reporting/trackers/gitlab/gitlab.go | 3 ++- v2/pkg/templates/cluster.go | 3 ++- v2/pkg/templates/types/types.go | 1 + v2/pkg/templates/workflows.go | 1 + 33 files changed, 54 insertions(+), 26 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8ff4dcb92..590b226be 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -275,6 +275,8 @@ import ( "path" "github.com/logrusorgru/aurora" + "go.uber.org/ratelimit" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" @@ -291,7 +293,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "go.uber.org/ratelimit" ) func main() { diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index a44aa4a60..d55f74e0f 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index e07c45c21..0a4446042 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index b5b0e5a34..d92058916 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/gobwas/ws/wsutil" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 92e6d47f9..cc38519b9 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -7,6 +7,7 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index f78400101..873a94e85 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -1,11 +1,12 @@ package core import ( + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" ) // Execute takes a list of templates/workflows that have been compiled diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index 406331fe0..901bc6b7f 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/filekv" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 8c255d7ad..9ee87df4a 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,11 +1,12 @@ package core import ( + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/atomic" ) // executeWorkflow runs a workflow on an input and returns true or false diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index 77909a89f..cd2ea09ba 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -1,8 +1,9 @@ package core import ( - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/remeh/sizedwaitgroup" + + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // WorkPool implements an execution pool for executing different diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index e4cee9c5e..9e1a9f2ad 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -19,10 +19,11 @@ import ( "time" "github.com/Knetic/govaluate" + "github.com/spaolacci/murmur3" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/spaolacci/murmur3" ) const ( diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index f75baf087..e5bfb0d64 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -8,8 +8,9 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestDSLURLEncodeDecode(t *testing.T) { diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index e43ea145e..e06428ac2 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,9 +1,8 @@ package extractors import ( - "strings" - "encoding/json" + "strings" "github.com/antchfx/htmlquery" diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go index ef74a317e..32a69ef5b 100644 --- a/v2/pkg/parsers/parser_test.go +++ b/v2/pkg/parsers/parser_test.go @@ -5,11 +5,12 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/stretchr/testify/require" ) func TestLoadTemplate(t *testing.T) { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index f3aaa8a34..07b3a3d0a 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -4,6 +4,7 @@ package generators import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 75e39d0ed..71a033e6c 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -3,8 +3,9 @@ package generators import ( "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" ) func TestBatteringRamGenerator(t *testing.T) { diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index 1877a1ab3..307c33a2f 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -2,6 +2,7 @@ package protocolinit import ( "github.com/corpix/uarand" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" diff --git a/v2/pkg/protocols/common/protocolstate/state.go b/v2/pkg/protocols/common/protocolstate/state.go index b43e40cf9..28c9df525 100644 --- a/v2/pkg/protocols/common/protocolstate/state.go +++ b/v2/pkg/protocols/common/protocolstate/state.go @@ -2,6 +2,7 @@ package protocolstate import ( "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 614159e2a..a1a7b5c37 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,8 +12,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" - "github.com/projectdiscovery/retryabledns" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/retryabledns" ) var _ protocols.Request = &Request{} diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index 1f40f98f7..e61803d4b 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -12,9 +12,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils" + "golang.org/x/net/proxy" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index a77e3c947..4719a5d24 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestBaseURLWithTemplatePrefs(t *testing.T) { diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 3fee8024c..ee8c524c0 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -3,9 +3,10 @@ package http import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/stretchr/testify/require" ) func TestRequestGeneratorPaths(t *testing.T) { diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 5eae57efe..834cc091d 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -11,11 +11,12 @@ import ( "strings" "github.com/pkg/errors" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" ) type redirectedResponse struct { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 7ad9d24aa..f288b717f 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -4,6 +4,7 @@ import ( "go.uber.org/ratelimit" "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 388886be2..b1db17040 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -10,6 +10,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" + "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 8d7e02179..8764f0ce3 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -3,11 +3,12 @@ package ssl import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" - "github.com/stretchr/testify/require" ) func TestSSLProtocol(t *testing.T) { diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index ab864f65b..53a35df3c 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -15,6 +15,7 @@ import ( "github.com/gobwas/ws" "github.com/gobwas/ws/wsutil" "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" diff --git a/v2/pkg/reporting/dedupe/dedupe_test.go b/v2/pkg/reporting/dedupe/dedupe_test.go index 5d77f35cd..eb5014425 100644 --- a/v2/pkg/reporting/dedupe/dedupe_test.go +++ b/v2/pkg/reporting/dedupe/dedupe_test.go @@ -5,8 +5,9 @@ import ( "os" "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/stretchr/testify/require" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" ) func TestDedupeDuplicates(t *testing.T) { diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 0959cc929..6e009f42b 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -3,15 +3,15 @@ package es import ( "bytes" "crypto/tls" + "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "net/http" "time" - "encoding/base64" - "encoding/json" - "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 92c162f8f..d8f000584 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -10,8 +10,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index c1438e2dc..dddfa2e65 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -3,9 +3,10 @@ package gitlab import ( "fmt" + "github.com/xanzy/go-gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/xanzy/go-gitlab" ) // Integration is a client for an issue tracker integration diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 1ec1e95cb..597711e17 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -3,6 +3,8 @@ package templates import ( "fmt" + "github.com/rs/xid" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -10,7 +12,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/rs/xid" ) // Cluster clusters a list of templates into a lesser number if possible based diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index d26ad7f08..0b50eb83d 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -7,6 +7,7 @@ import ( "github.com/alecthomas/jsonschema" "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" ) diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 89f8d1bda..9de1f2686 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -2,6 +2,7 @@ package templates import ( "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" From fdaa8e4539b38c8980432d8c8ec3bbcebdce71ae Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:18:54 +0200 Subject: [PATCH 162/196] refactor: removed redundant type conversion --- v2/cmd/integration-test/http.go | 4 +- v2/cmd/integration-test/loader.go | 16 +-- v2/cmd/integration-test/websocket.go | 2 +- .../headless/engine/page_actions_test.go | 118 +++++++++--------- .../offlinehttp/read_response_test.go | 4 +- 5 files changed, 72 insertions(+), 72 deletions(-) diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 1d653afa2..1890048d2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -43,14 +43,14 @@ type httpInteractshRequest struct{} // Execute executes a test case and returns an error if occurred func (h *httpInteractshRequest) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { value := r.Header.Get("url") if value != "" { if resp, _ := http.DefaultClient.Get(value); resp != nil { resp.Body.Close() } } - })) + }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/cmd/integration-test/loader.go b/v2/cmd/integration-test/loader.go index 0a4446042..3507b8a31 100644 --- a/v2/cmd/integration-test/loader.go +++ b/v2/cmd/integration-test/loader.go @@ -25,14 +25,14 @@ type remoteTemplateList struct{} func (h *remoteTemplateList) Execute(templateList string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test matcher text") if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) - router.GET("/template_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/template_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { file, err := os.ReadFile(templateList) if err != nil { w.WriteHeader(500) @@ -41,7 +41,7 @@ func (h *remoteTemplateList) Execute(templateList string) error { if err != nil { w.WriteHeader(500) } - })) + }) ts := httptest.NewServer(router) defer ts.Close() @@ -61,14 +61,14 @@ type remoteWorkflowList struct{} func (h *remoteWorkflowList) Execute(workflowList string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test matcher text") if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) - router.GET("/workflow_list", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + router.GET("/workflow_list", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { file, err := os.ReadFile(workflowList) if err != nil { w.WriteHeader(500) @@ -77,7 +77,7 @@ func (h *remoteWorkflowList) Execute(workflowList string) error { if err != nil { w.WriteHeader(500) } - })) + }) ts := httptest.NewServer(router) defer ts.Close() diff --git a/v2/cmd/integration-test/websocket.go b/v2/cmd/integration-test/websocket.go index d92058916..af6d07451 100644 --- a/v2/cmd/integration-test/websocket.go +++ b/v2/cmd/integration-test/websocket.go @@ -23,7 +23,7 @@ func (h *websocketBasic) Execute(filePath string) error { connHandler := func(conn net.Conn) { for { msg, op, _ := wsutil.ReadClientData(conn) - if string(msg) != string("hello") { + if string(msg) != "hello" { return } _ = wsutil.WriteServerMessage(conn, op, []byte("world")) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index f9819756d..4c3414e47 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) { ` - actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}} + actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}} testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -50,9 +50,9 @@ func TestActionScript(t *testing.T) { t.Run("run-and-results", func(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { @@ -64,10 +64,10 @@ func TestActionScript(t *testing.T) { t.Run("hook", func(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") @@ -88,9 +88,9 @@ func TestActionClick(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -121,9 +121,9 @@ func TestActionRightClick(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionRightClick}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -146,9 +146,9 @@ func TestActionTextInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionTextInput}, Data: map[string]string{"selector": "input", "value": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -163,9 +163,9 @@ func TestActionTextInput(t *testing.T) { func TestActionHeadersChange(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetHeader}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -190,9 +190,9 @@ func TestActionScreenshot(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionScreenshot}, Data: map[string]string{"to": "test"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -215,9 +215,9 @@ func TestActionTimeInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionTimeInput}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -243,9 +243,9 @@ func TestActionSelectInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionSelectInput}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -266,9 +266,9 @@ func TestActionFilesInput(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionFilesInput}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -292,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -317,8 +317,8 @@ func TestActionGetResource(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionGetResource}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -337,8 +337,8 @@ func TestActionExtract(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionExtract}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -356,8 +356,8 @@ func TestActionSetMethod(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetMethod}, Data: map[string]string{"part": "x", "method": "SET"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -368,9 +368,9 @@ func TestActionSetMethod(t *testing.T) { func TestActionAddHeader(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -387,11 +387,11 @@ func TestActionAddHeader(t *testing.T) { func TestActionDeleteHeader(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionAddHeader}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: ActionTypeHolder{ActionType: ActionDeleteHeader}, Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -408,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) { func TestActionSetBody(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, + {ActionType: ActionTypeHolder{ActionType: ActionSetBody}, Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, } handler := func(w http.ResponseWriter, r *http.Request) { @@ -436,10 +436,10 @@ func TestActionKeyboard(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, + {ActionType: ActionTypeHolder{ActionType: ActionClick}, Data: map[string]string{"selector": "input"}}, + {ActionType: ActionTypeHolder{ActionType: ActionKeyboard}, Data: map[string]string{"keys": "Test2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -462,8 +462,8 @@ func TestActionSleep(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionSleep}, Data: map[string]string{"duration": "2"}}, } testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { @@ -485,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) { ` actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: ActionTypeHolder{ActionType: ActionWaitVisible}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, } t.Run("wait for an element being visible", func(t *testing.T) { diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index f47b345dc..a26f2371a 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -159,7 +159,7 @@ Server: Google Frontend t.Run("test-live-response-with-content-length", func(t *testing.T) { var ts *httptest.Server router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + router.GET("/", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Server", "Google Frontend") fmt.Fprintf(w, "%s", ` @@ -172,7 +172,7 @@ Server: Google Frontend

`) - })) + }) ts = httptest.NewServer(router) defer ts.Close() From ec6889931d50fca916c97a44fad3e251d9baa439 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 17:57:22 +0200 Subject: [PATCH 163/196] refactor: linter driven fixes * x = x + "" => x += "" * pre-allocating slice with known size * added t.Helper() methods in test helpers * complex if-else conditions replaced by switches * errors should be checked using error.Is() instead of == * function parameter should start with lower case letter * removed unnecessary type definition * variable/label naming convention: camelCase instead of snake_case --- v2/cmd/cve-annotate/main.go | 10 ++++----- v2/internal/runner/update.go | 4 ++-- v2/pkg/model/model_test.go | 1 + v2/pkg/model/types/severity/severities.go | 2 +- .../model/types/severity/severity_holder.go | 1 - v2/pkg/model/types/severity/severity_test.go | 2 ++ v2/pkg/model/types/stringslice/stringslice.go | 7 ++++--- v2/pkg/protocols/common/generators/load.go | 2 +- .../common/hosterrorscache/hosterrorscache.go | 5 ++--- v2/pkg/protocols/headless/engine/rules.go | 21 +++++++++++-------- .../http/httpclientpool/clientpool.go | 2 +- v2/pkg/protocols/network/request.go | 4 ++-- v2/pkg/utils/yaml/yaml_decode_wrapper.go | 2 +- 13 files changed, 34 insertions(+), 29 deletions(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index 45ede0420..c2ccaa765 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -117,21 +117,21 @@ func getCVEData(client *nvd.Client, filePath, data string) { } if !strings.Contains(infoBlockClean, "classification") && (cvssScore != 0 && cvssMetrics != "") { changed = true - newInfoBlock = newInfoBlock + fmt.Sprintf("\n classification:\n cvss-metrics: %s\n cvss-score: %.2f\n cve-id: %s", cvssMetrics, cvssScore, cveName) + newInfoBlock += fmt.Sprintf("\n classification:\n cvss-metrics: %s\n cvss-score: %.2f\n cve-id: %s", cvssMetrics, cvssScore, cveName) if len(cweID) > 0 && (cweID[0] != "NVD-CWE-Other" && cweID[0] != "NVD-CWE-noinfo") { - newInfoBlock = newInfoBlock + fmt.Sprintf("\n cwe-id: %s", strings.Join(cweID, ",")) + newInfoBlock += fmt.Sprintf("\n cwe-id: %s", strings.Join(cweID, ",")) } } // If there is no description field, fill the description from CVE information if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 { changed = true - newInfoBlock = newInfoBlock + fmt.Sprintf("\n description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value)) + newInfoBlock += fmt.Sprintf("\n description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value)) } if !strings.Contains(infoBlockClean, "reference:") && len(cveItem.CVE.References.ReferenceData) > 0 { changed = true - newInfoBlock = newInfoBlock + "\n reference:" + newInfoBlock += "\n reference:" for _, reference := range cveItem.CVE.References.ReferenceData { - newInfoBlock = newInfoBlock + fmt.Sprintf("\n - %s", reference.URL) + newInfoBlock += fmt.Sprintf("\n - %s", reference.URL) } } newTemplate := strings.ReplaceAll(data, infoBlockClean, newInfoBlock) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 8a0fa6c5a..99d94cc88 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -333,7 +333,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat isAddition = true } - newTemplateChecksum, err := writeUnZippedTemplateFile(err, templateAbsolutePath, zipTemplateFile) + newTemplateChecksum, err := writeUnZippedTemplateFile(templateAbsolutePath, zipTemplateFile) if err != nil { return nil, err } @@ -366,7 +366,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat return results, nil } -func writeUnZippedTemplateFile(err error, templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { +func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip.File) (string, error) { templateFile, err := os.OpenFile(templateAbsolutePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return "", fmt.Errorf("could not create template file: %w", err) diff --git a/v2/pkg/model/model_test.go b/v2/pkg/model/model_test.go index bdbe27ddb..e0803c31a 100644 --- a/v2/pkg/model/model_test.go +++ b/v2/pkg/model/model_test.go @@ -72,6 +72,7 @@ func TestUnmarshal(t *testing.T) { } assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info { + t.Helper() info := Info{} err := yaml.Unmarshal([]byte(yamlPayload), &info) assert.Nil(t, err) diff --git a/v2/pkg/model/types/severity/severities.go b/v2/pkg/model/types/severity/severities.go index 9e3244859..91ffa1ceb 100644 --- a/v2/pkg/model/types/severity/severities.go +++ b/v2/pkg/model/types/severity/severities.go @@ -43,7 +43,7 @@ func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) e } func (severities Severities) String() string { - var stringSeverities []string + var stringSeverities = make([]string, 0, len(severities)) for _, severity := range severities { stringSeverities = append(stringSeverities, severity.String()) } diff --git a/v2/pkg/model/types/severity/severity_holder.go b/v2/pkg/model/types/severity/severity_holder.go index ad4c2496d..7d52f9c81 100644 --- a/v2/pkg/model/types/severity/severity_holder.go +++ b/v2/pkg/model/types/severity/severity_holder.go @@ -6,7 +6,6 @@ import ( "github.com/alecthomas/jsonschema" ) -//nolint:exported,revive //prefer to be explicit about the name, and make it refactor-safe // Holder holds a Severity type. Required for un/marshalling purposes type Holder struct { Severity Severity diff --git a/v2/pkg/model/types/severity/severity_test.go b/v2/pkg/model/types/severity/severity_test.go index b21f57265..6ba472388 100644 --- a/v2/pkg/model/types/severity/severity_test.go +++ b/v2/pkg/model/types/severity/severity_test.go @@ -30,6 +30,7 @@ func TestGetSupportedSeverities(t *testing.T) { } func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) { + t.Helper() payloads := [...]string{ payloadCreator("Info"), payloadCreator("info"), @@ -48,6 +49,7 @@ func testUnmarshal(t *testing.T, unmarshaller func(data []byte, v interface{}) e } func testUnmarshalFail(t *testing.T, unmarshaller func(data []byte, v interface{}) error, payloadCreator func(value string) string) { + t.Helper() assert.Panics(t, func() { unmarshal(payloadCreator("invalid"), unmarshaller) }) } diff --git a/v2/pkg/model/types/stringslice/stringslice.go b/v2/pkg/model/types/stringslice/stringslice.go index 4a3e28486..55d798550 100644 --- a/v2/pkg/model/types/stringslice/stringslice.go +++ b/v2/pkg/model/types/stringslice/stringslice.go @@ -80,11 +80,12 @@ func marshalStringToSlice(unmarshal func(interface{}) error) ([]string, error) { } var result []string - if len(marshalledValuesAsSlice) > 0 { + switch { + case len(marshalledValuesAsSlice) > 0: result = marshalledValuesAsSlice - } else if utils.IsNotBlank(marshalledValueAsString) { + case utils.IsNotBlank(marshalledValueAsString): result = strings.Split(marshalledValueAsString, ",") - } else { + default: result = []string{} } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 0c44b613c..d5b00e4d3 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -53,7 +53,7 @@ func loadPayloadsFromFile(filepath string) ([]string, error) { } lines = append(lines, text) } - if err := scanner.Err(); err != nil && err != io.EOF { + if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) { return lines, scanner.Err() } return lines, nil diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 0abf1b524..c509ff847 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -25,11 +25,11 @@ type Cache struct { const DefaultMaxHostsCount = 10000 // New returns a new host max errors cache -func New(MaxHostError, maxHostsCount int) *Cache { +func New(maxHostError, maxHostsCount int) *Cache { gc := gcache.New(maxHostsCount). ARC(). Build() - return &Cache{failedTargets: gc, MaxHostError: MaxHostError} + return &Cache{failedTargets: gc, MaxHostError: maxHostError} } // SetVerbose sets the cache to log at verbose level @@ -47,7 +47,6 @@ func (c *Cache) normalizeCacheValue(value string) string { finalValue := value if strings.HasPrefix(value, "http") { if parsed, err := url.Parse(value); err == nil { - hostname := parsed.Host finalPort := parsed.Port() if finalPort == "" { diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a802a64a9..0c5359415 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -16,15 +16,16 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - if rule.Action == ActionSetMethod { + switch { + case rule.Action == ActionSetMethod: ctx.Request.Req().Method = rule.Args["method"] - } else if rule.Action == ActionAddHeader { + case rule.Action == ActionAddHeader: ctx.Request.Req().Header.Add(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionSetHeader { + case rule.Action == ActionSetHeader: ctx.Request.Req().Header.Set(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionDeleteHeader { + case rule.Action == ActionDeleteHeader: ctx.Request.Req().Header.Del(rule.Args["key"]) - } else if rule.Action == ActionSetBody { + case rule.Action == ActionSetBody: body := rule.Args["body"] ctx.Request.Req().ContentLength = int64(len(body)) ctx.Request.SetBody(body) @@ -36,13 +37,15 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { if rule.Part != "response" { continue } - if rule.Action == ActionAddHeader { + + switch { + case rule.Action == ActionAddHeader: ctx.Response.Headers().Add(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionSetHeader { + case rule.Action == ActionSetHeader: ctx.Response.Headers().Set(rule.Args["key"], rule.Args["value"]) - } else if rule.Action == ActionDeleteHeader { + case rule.Action == ActionDeleteHeader: ctx.Response.Headers().Del(rule.Args["key"]) - } else if rule.Action == ActionSetBody { + case rule.Action == ActionSetBody: body := rule.Args["body"] ctx.Response.Headers().Set("Content-Length", fmt.Sprintf("%d", len(body))) ctx.Response.SetBody(rule.Args["body"]) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 191ca3cf0..9305870a6 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -185,7 +185,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if proxyURL != nil { // Attempts to overwrite the dial function with the socks proxied version if proxyURL.Scheme == types.SOCKS5 { - var proxyAuth *proxy.Auth = &proxy.Auth{} + var proxyAuth = &proxy.Auth{} proxyAuth.User = proxyURL.User.Username() proxyAuth.Password, _ = proxyURL.User.Password() diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 35da81d15..e4c0ac410 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -223,12 +223,12 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input <-t.C } } - read_socket: + readSocket: for { select { case <-readInterval.C: closeTimer(readInterval) - break read_socket + break readSocket default: buf := make([]byte, bufferSize) nBuf, err := conn.Read(buf) diff --git a/v2/pkg/utils/yaml/yaml_decode_wrapper.go b/v2/pkg/utils/yaml/yaml_decode_wrapper.go index a9cb422bb..3bc4fa605 100644 --- a/v2/pkg/utils/yaml/yaml_decode_wrapper.go +++ b/v2/pkg/utils/yaml/yaml_decode_wrapper.go @@ -19,8 +19,8 @@ func DecodeAndValidate(r io.Reader, v interface{}) error { if validate == nil { validate = validator.New() } - if err := validate.Struct(v); err != nil { + if err := validate.Struct(v); err != nil { if _, ok := err.(*validator.InvalidValidationError); ok { return err } From 7e22d70dedcab5e032426c14d1a89256eac2c1d6 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Thu, 25 Nov 2021 18:54:16 +0200 Subject: [PATCH 164/196] refactor/documentation: typos and grammatical errors --- .github/ISSUE_TEMPLATE/issue-report.md | 2 +- DESIGN.md | 36 +++++++++--------- README.md | 4 +- .../test-issue-tracker-config1.yaml | 30 +++++++-------- .../test-issue-tracker-config2.yaml | 38 +++++++++---------- v2/cmd/nuclei/issue-tracker-config.yaml | 36 +++++++++--------- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 2 +- v2/pkg/catalog/find.go | 2 +- v2/pkg/catalog/loader/filter/tag_filter.go | 6 +-- v2/pkg/catalog/loader/loader.go | 6 +-- v2/pkg/core/execute.go | 8 ++-- v2/pkg/core/workpool.go | 14 +++---- v2/pkg/operators/matchers/matchers.go | 2 +- v2/pkg/projectfile/httputil.go | 28 -------------- v2/pkg/projectfile/project.go | 16 ++++---- .../common/generators/attack_types.go | 8 ++-- .../protocols/common/generators/generators.go | 6 +-- .../common/generators/generators_test.go | 2 +- .../protocols/common/generators/validate.go | 16 ++++---- .../hosterrorscache/hosterrorscache_test.go | 6 +-- .../protocols/dns/dnsclientpool/clientpool.go | 2 +- v2/pkg/protocols/dns/operators.go | 10 ++--- v2/pkg/protocols/dns/request.go | 16 ++++---- v2/pkg/protocols/headless/engine/engine.go | 10 ++--- .../protocols/headless/engine/http_client.go | 4 +- .../protocols/headless/engine/page_actions.go | 12 +++--- v2/pkg/protocols/headless/engine/rules.go | 2 +- v2/pkg/protocols/http/build_request_test.go | 6 +-- v2/pkg/protocols/http/http.go | 6 +-- v2/pkg/protocols/http/http_test.go | 2 +- .../http/httpclientpool/clientpool.go | 22 +++++------ .../protocols/http/race/syncedreadcloser.go | 12 +++--- v2/pkg/protocols/http/raw/raw.go | 6 +-- v2/pkg/protocols/http/request.go | 6 +-- .../protocols/http/request_generator_test.go | 4 +- v2/pkg/protocols/http/utils.go | 8 ++-- v2/pkg/protocols/network/network.go | 6 +-- v2/pkg/protocols/websocket/websocket.go | 2 +- .../reporting/exporters/es/elasticsearch.go | 2 +- v2/pkg/reporting/reporting.go | 16 ++++---- v2/pkg/reporting/trackers/github/github.go | 8 ++-- v2/pkg/templates/templates_doc.go | 2 +- v2/pkg/testutils/testutils.go | 2 +- v2/pkg/types/types.go | 6 +-- v2/pkg/utils/stats/stats.go | 4 +- 46 files changed, 209 insertions(+), 237 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md index aeec33c17..264b09ccc 100644 --- a/.github/ISSUE_TEMPLATE/issue-report.md +++ b/.github/ISSUE_TEMPLATE/issue-report.md @@ -33,4 +33,4 @@ Example: steps to reproduce the behavior: ### Anything else: - + diff --git a/DESIGN.md b/DESIGN.md index 590b226be..31f3a2b99 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -10,9 +10,9 @@ Template is the basic unit of input to the engine which describes the requests t The template structure is described here. Template level attributes are defined here as well as convenience methods to validate, parse and compile templates creating executers. -Any attributes etc required for the template, engine or requests to function are also set here. +Any attributes etc. required for the template, engine or requests to function are also set here. -Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc on the paths provided are also done here. +Workflows are also compiled, their templates are loaded and compiled as well. Any validations etc. on the paths provided are also done here. `Parse` function is the main entry point which returns a template for a `filePath` and `executorOptions`. It compiles all the requests for the templates, all the workflows, as well as any self-contained request etc. It also caches the templates in an in-memory cache. @@ -106,7 +106,7 @@ The default executer is provided in `pkg/protocols/common/executer` . It takes a A different executer is the Clustered Requests executer which implements the Nuclei Request clustering functionality in `pkg/templates` We have a single HTTP request in cases where multiple templates can be clustered and multiple operator lists to match/extract. The first HTTP request is executed while all the template matcher/extractor are evaluated separately. -For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently from the template execution. +For Workflow execution, a separate RunWorkflow function is used which executes the workflow independently of the template execution. With this basic premise set, we can now start exploring the current runner implementation which will also walk us through the architecture of nuclei. @@ -118,7 +118,7 @@ The first process after all CLI specific initialisation is the loading of templa #### pkg/catalog -This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template directory as well as the current user directory. +This package is used to get paths using mixed syntax. It takes a template directory and performs resolving for template paths both from provided template and current user directory. The syntax is very versatile and can include filenames, glob patterns, directories, absolute paths, and relative-paths. @@ -177,7 +177,7 @@ ResultEvent structure is passed to the Nuclei Output Writer which contains the e #### pkg/protocols/common/interactsh -Interactsh module is used to provide automatic Out of Band vulnerability identification in Nuclei. +Interactsh module is used to provide automatic Out-of-Band vulnerability identification in Nuclei. It uses two LRU caches, one for storing interactions for request URLs and one for storing requests for interaction URL. These both caches are used to correlated requests received to the Interactsh OOB server and Nuclei Instance. [Interactsh Client](https://github.com/projectdiscovery/interactsh/pkg/client) package does most of the heavy lifting of this module. @@ -193,13 +193,13 @@ Next we arrive in the `RunEnumeration` function of the runner. Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `v2/pkg/parsers/workflow_loader.go` -The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc to return compiled `Templates` and `Workflows`. +The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc. to return compiled `Templates` and `Workflows`. #### pkg/catalog/loader -First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to removed the excluded template/workflows paths. +First the input passed by the user as paths is normalised to absolute paths which is done by the `pkg/catalog` module. Next the path filter module is used to remove the excluded template/workflows paths. -`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. +`pkg/parsers` module's `LoadTemplate`,`LoadWorkflow` functions are used to check if the templates pass the validation + are not excluded via tags/severity/etc. filters. If all checks are passed, then the template/workflow is parsed and returned in a compiled form by the `pkg/templates`'s `Parse` function. `Parse` function performs compilation of all the requests in a template + creates Executers from them returning a runnable Template/Workflow structure. @@ -207,10 +207,10 @@ Clustering module comes in next whose job is to cluster identical HTTP GET reque ### pkg/operators -Operators package implements all of the matching and extracting logic of Nuclei. +Operators package implements all the matching and extracting logic of Nuclei. ```go -// Operators contains the operators that can be applied on protocols +// Operators contain the operators that can be applied on protocols type Operators struct { Matchers []*matchers.Matcher Extractors []*extractors.Extractor @@ -218,7 +218,7 @@ type Operators struct { } ``` -A protocol only needs to embed the `operators.Operators` type shown above and it can utilise all the matching/extracting functionality of nuclei. +A protocol only needs to embed the `operators.Operators` type shown above, and it can utilise all the matching/extracting functionality of nuclei. ```go // MatchFunc performs matching operation for a matcher on model and returns true or false. @@ -246,7 +246,7 @@ type Result struct { } ``` -The internal logics for matching and extracting for things like words, regexes, jq, paths, etc is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. +The internal logics for matching and extracting for things like words, regexes, jq, paths, etc. is specified in `pkg/operators/matchers`, `pkg/operators/extractors`. Those packages should be investigated for further look into the topic. ### Template Execution @@ -357,7 +357,7 @@ func main() { ### Adding a New Protocol -Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc are implemented in form of protocol requests. +Protocols form the core of Nuclei Engine. All the request types like `http`, `dns`, etc. are implemented in form of protocol requests. A protocol must implement the `Protocol` and `Request` interfaces described above in `pkg/protocols`. We'll take the example of an existing protocol implementation - websocket for this short reference around Nuclei internals. @@ -475,13 +475,13 @@ func (r *Request) Type() templateTypes.ProtocolType { } ``` -Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. +Almost all of these protocols have boilerplate functions for which default implementations have been provided in the `providers` package. Examples are the implementation of `Match`, `Extract`, `MakeResultEvent`, GetCompiledOperators`, etc. which are almost same throughout Nuclei protocols code. It is enough to copy-paste them unless customization is required. `eventcreator` package offers `CreateEventWithAdditionalOptions` function which can be used to create result events after doing request execution. Step by step description of how to add a new protocol to Nuclei - -1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with less number of options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. +1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with fewer options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`. 2. Add the protocol request structure to the `Template` structure fields. This is done in `pkg/templates/templates.go` with the corresponding import line. @@ -527,7 +527,7 @@ func (t *Template) Type() templateTypes.ProtocolType { ```go -// Requests returns the total request count for the template +// Requests return the total request count for the template func (template *Template) Requests() int { return len(template.RequestsDNS) + ... @@ -559,7 +559,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results - [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - Gitlab Issue Tracker Exporter - [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter -- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - Github Issue Tracker Exporter +- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter - [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions - [v2/pkg/parsers](./v2/pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading. - [v2/pkg/types](./v2/pkg/types) - Contains CLI options as well as misc helper functions. @@ -583,7 +583,7 @@ That's it, you've added a new protocol to Nuclei. The next good step would be to - [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol - [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support - [v2/pkg/protocols/common/interactsh](./v2/pkg/protocols/common/interactsh) - Interactsh integration -- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc) +- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc.) - [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer - [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers - [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator diff --git a/README.md b/README.md index efe80f381..427b11e97 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest ### Nuclei Templates -Nuclei has had built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated. +Nuclei has built-in support for automatic template download/update as default since version [v2.5.2](https://github.com/projectdiscovery/nuclei/releases/tag/v2.5.2). [**Nuclei-Templates**](https://github.com/projectdiscovery/nuclei-templates) project provides a community-contributed list of ready-to-use templates that is constantly updated. You may still use the `update-templates` flag to update the nuclei templates at any time; You can write your own checks for your individual workflow and needs following Nuclei's [templating guide](https://nuclei.projectdiscovery.io/templating-guide/). @@ -250,7 +250,7 @@ Please check our other open-source projects that might fit into your bug bounty Nuclei immensely improve how you approach security assessment by augmenting the manual, repetitive processes. Consultancies are already converting their manual assessment steps with Nuclei, it allows them to run set of their custom assessment approach across thousands of hosts in an automated manner. -Pen-testers get the full power of our public templates and customization capabilities to speed-up their assessment process, and specifically with the regression cycle where you can easily verify the fix. +Pen-testers get the full power of our public templates and customization capabilities to speed up their assessment process, and specifically with the regression cycle where you can easily verify the fix. - Easily create your compliance, standards suite (e.g. OWASP Top 10) checklist. - With capabilities like [fuzz](https://nuclei.projectdiscovery.io/templating-guide/#advance-fuzzing) and [workflows](https://nuclei.projectdiscovery.io/templating-guide/#workflows), complex manual steps and repetitive assessment can be easily automated with Nuclei. diff --git a/integration_tests/test-issue-tracker-config1.yaml b/integration_tests/test-issue-tracker-config1.yaml index dd8ceb180..2f8b587f7 100644 --- a/integration_tests/test-issue-tracker-config1.yaml +++ b/integration_tests/test-issue-tracker-config1.yaml @@ -3,35 +3,35 @@ allow-list: deny-list: severity: low -# github contains configuration options for github issue tracker +# GitHub contains configuration options for GitHub issue tracker github: - # base-url is the optional self-hosted github application url + # base-url is the optional self-hosted GitHub application url base-url: https://localhost:8443/github - # username is the username of the github user + # username is the username of the GitHub user username: test-username - # owner is the owner name of the repository for issues. + # owner is the owner name of the repository for issues owner: test-owner - # token is the token for github account. + # token is the token for GitHub account token: test-token - # project-name is the name of the repository. + # project-name is the name of the repository project-name: test-project # issue-label is the label of the created issue type issue-label: bug -# gitlab contains configuration options for gitlab issue tracker +# GitLab contains configuration options for gitlab issue tracker gitlab: - # base-url is the optional self-hosted gitlab application url + # base-url is the optional self-hosted GitLab application url base-url: https://localhost:8443/gitlab - # username is the username of the gitlab user + # username is the username of the GitLab user username: test-username - # token is the token for gitlab account. + # token is the token for GitLab account token: test-token - # project-name is the name/id of the project(repository). + # project-name is the name/id of the project(repository) project-name: "1234" # issue-label is the label of the created issue type issue-label: bug -# jira contains configuration options for jira issue tracker +# Jira contains configuration options for Jira issue tracker jira: # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used cloud: true @@ -39,11 +39,11 @@ jira: update-existing: false # URL is the jira application url url: https://localhost/jira - # account-id is the account-id of the jira user or username in case of on-prem Jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira account-id: test-account-id - # email is the email of the user for jira instance + # email is the email of the user for Jira instance email: test@test.com - # token is the token for jira instance or password in case of on-prem Jira + # token is the token for Jira instance or password in case of on-prem Jira token: test-token # project-name is the name of the project. project-name: test-project-name diff --git a/integration_tests/test-issue-tracker-config2.yaml b/integration_tests/test-issue-tracker-config2.yaml index c76b773eb..f548dbfbd 100644 --- a/integration_tests/test-issue-tracker-config2.yaml +++ b/integration_tests/test-issue-tracker-config2.yaml @@ -5,47 +5,47 @@ allow-list: deny-list: severity: low -# github contains configuration options for github issue tracker -github: - # base-url is the optional self-hosted github application url - base-url: https://localhost:8443/github - # username is the username of the github user +# GitHub contains configuration options for GitHub issue tracker +GitHub: + # base-url is the optional self-hosted GitHub application url + base-url: https://localhost:8443/GitHub + # username is the username of the GitHub user username: test-username # owner is the owner name of the repository for issues. owner: test-owner - # token is the token for github account. + # token is the token for GitHub account. token: test-token # project-name is the name of the repository. project-name: test-project # issue-label is the label of the created issue type issue-label: bug -# gitlab contains configuration options for gitlab issue tracker -gitlab: - # base-url is the optional self-hosted gitlab application url - base-url: https://localhost:8443/gitlab - # username is the username of the gitlab user +# GitLab contains configuration options for GitLab issue tracker +GitLab: + # base-url is the optional self-hosted GitLab application url + base-url: https://localhost:8443/GitLab + # username is the username of the GitLab user username: test-username - # token is the token for gitlab account. + # token is the token for GitLab account. token: test-token # project-name is the name/id of the project(repository). project-name: "1234" # issue-label is the label of the created issue type issue-label: bug -# jira contains configuration options for jira issue tracker -jira: +# Jira contains configuration options for Jira issue tracker +Jira: # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used cloud: true # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created update-existing: false - # URL is the jira application url - url: https://localhost/jira - # account-id is the account-id of the jira user or username in case of on-prem Jira + # URL is the Jira application url + url: https://localhost/Jira + # account-id is the account-id of the Jira user or username in case of on-prem Jira account-id: test-account-id - # email is the email of the user for jira instance + # email is the email of the user for Jira instance email: test@test.com - # token is the token for jira instance or password in case of on-prem Jira + # token is the token for Jira instance or password in case of on-prem Jira token: test-token # project-name is the name of the project. project-name: test-project-name diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index 5364db5f5..508446243 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -5,51 +5,51 @@ #deny-list: # severity: info, low, medium -# github contains configuration options for github issue tracker -#github: -# # base-url (optional) is the self-hosted github application url +# GitHub contains configuration options for GitHub issue tracker +#GitHub: +# # base-url (optional) is the self-hosted GitHub application url # base-url: "" -# # username is the username of the github user +# # username is the username of the GitHub user # username: "" # # owner is the owner name of the repository for issues. # owner: "" -# # token is the token for github account. +# # token is the token for GitHub account. # token: "" # # project-name is the name of the repository. # project-name: "" # # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the severity as the label of the created issue type # severity-as-label: false -# gitlab contains configuration options for gitlab issue tracker -#gitlab: -# # base-url (optional) is the self-hosted gitlab application url +# GitLab contains configuration options for GitLab issue tracker +#GitLab: +# # base-url (optional) is the self-hosted GitLab application url # base-url: "" -# # username is the username of the gitlab user +# # username is the username of the GitLab user # username: "" -# # token is the token for gitlab account. +# # token is the token for GitLab account. # token: "" # # project-id is the ID of the repository. # project-id: "" # # issue-label (optional) is the label of the created issue type # issue-label: "" -# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# # severity-as-label (optional) sets the severity as the label of the created issue type # severity-as-label: false -# jira contains configuration options for jira issue tracker -#jira: +# Jira contains configuration options for Jira issue tracker +#Jira: # # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true # # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false -# # URL is the jira application url +# # URL is the Jira application URL # url: "" -# # account-id is the account-id of the jira user or username in case of on-prem Jira +# # account-id is the account-id of the Jira user or username in case of on-prem Jira # account-id: "" -# # email is the email of the user for jira instance +# # email is the email of the user for Jira instance # email: "" -# # token is the token for jira instance or password in case of on-prem Jira +# # token is the token for Jira instance or password in case of on-prem Jira # token: "" # # project-name is the name of the project. # project-name: "" diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 116479732..059a96167 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -110,7 +110,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), - flagSet.IntVar(&options.InteractionsCooldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.IntVar(&options.InteractionsCoolDownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 055eca1ab..a5a74f773 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -151,7 +151,7 @@ func New(options *types.Options) (*Runner, error) { opts.Authorization = options.InteractshToken opts.CacheSize = int64(options.InteractionsCacheSize) opts.Eviction = time.Duration(options.InteractionsEviction) * time.Second - opts.ColldownPeriod = time.Duration(options.InteractionsCooldownPeriod) * time.Second + opts.ColldownPeriod = time.Duration(options.InteractionsCoolDownPeriod) * time.Second opts.PollDuration = time.Duration(options.InteractionsPollDuration) * time.Second opts.NoInteractsh = runner.options.NoInteractsh diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index cc38519b9..7b3ffc6b4 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -80,7 +80,7 @@ func (c *Catalog) GetTemplatePath(target string) ([]string, error) { } // convertPathToAbsolute resolves the paths provided to absolute paths -// before doing any operations on them regardless of them being blob, folders, files, etc. +// before doing any operations on them regardless of them being BLOB, folders, files, etc. func (c *Catalog) convertPathToAbsolute(t string) (string, error) { if strings.Contains(t, "*") { file := filepath.Base(t) diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index 1420003ec..271b8e207 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -230,9 +230,9 @@ func splitCommaTrim(value string) []string { if !strings.Contains(value, ",") { return []string{strings.ToLower(value)} } - splitted := strings.Split(value, ",") - final := make([]string, len(splitted)) - for i, value := range splitted { + split := strings.Split(value, ",") + final := make([]string, len(split)) + for i, value := range split { final[i] = strings.ToLower(strings.TrimSpace(value)) } return final diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 1a61cb25b..ff8121588 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -98,8 +98,8 @@ func New(config *Config) (*Store, error) { finalWorkflows: config.Workflows, } - urlbasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 - if urlbasedTemplatesProvided { + urlBasedTemplatesProvided := len(config.TemplateURLs) > 0 || len(config.WorkflowURLs) > 0 + if urlBasedTemplatesProvided { remoteTemplates, remoteWorkflows, err := getRemoteTemplatesAndWorkflows(config.TemplateURLs, config.WorkflowURLs) if err != nil { return store, err @@ -109,7 +109,7 @@ func New(config *Config) (*Store, error) { } // Handle a case with no templates or workflows, where we use base directory - if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlbasedTemplatesProvided { + if len(store.finalTemplates) == 0 && len(store.finalWorkflows) == 0 && !urlBasedTemplatesProvided { store.finalTemplates = []string{config.TemplatesDirectory} } diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 873a94e85..60b6c20bf 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -18,7 +18,7 @@ func (e *Engine) Execute(templates []*templates.Template, target InputProvider) return e.ExecuteWithOpts(templates, target, false) } -// ExecuteWithOpts is execute with the full options +// ExecuteWithOpts executes with the full options func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target InputProvider, noCluster bool) *atomic.Bool { var finalTemplates []*templates.Template if !noCluster { @@ -72,9 +72,9 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template return } - wg.Waitgroup.Add() + wg.WaitGroup.Add() go func(value string) { - defer wg.Waitgroup.Done() + defer wg.WaitGroup.Done() var match bool var err error @@ -90,5 +90,5 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template results.CAS(false, match) }(scannedValue) }) - wg.Waitgroup.Wait() + wg.WaitGroup.Wait() } diff --git a/v2/pkg/core/workpool.go b/v2/pkg/core/workpool.go index cd2ea09ba..46ca8549d 100644 --- a/v2/pkg/core/workpool.go +++ b/v2/pkg/core/workpool.go @@ -7,7 +7,7 @@ import ( ) // WorkPool implements an execution pool for executing different -// types of task with different concurreny requirements. +// types of task with different concurrency requirements. // // It also allows Configuration of such requirements. This is used // for per-module like separate headless concurrency etc. @@ -17,7 +17,7 @@ type WorkPool struct { config WorkPoolConfig } -// WorkPoolConfig is the configuration for workpool +// WorkPoolConfig is the configuration for work pool type WorkPoolConfig struct { // InputConcurrency is the concurrency for inputs values. InputConcurrency int @@ -41,18 +41,18 @@ func NewWorkPool(config WorkPoolConfig) *WorkPool { } } -// Wait waits for all the workpool waitgroups to finish +// Wait waits for all the work pool wait groups to finish func (w *WorkPool) Wait() { w.Default.Wait() w.Headless.Wait() } -// InputWorkPool is a workpool per-input +// InputWorkPool is a work pool per-input type InputWorkPool struct { - Waitgroup *sizedwaitgroup.SizedWaitGroup + WaitGroup *sizedwaitgroup.SizedWaitGroup } -// InputPool returns a workpool for an input type +// InputPool returns a work pool for an input type func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { var count int if templateType == types.HeadlessProtocol { @@ -61,5 +61,5 @@ func (w *WorkPool) InputPool(templateType types.ProtocolType) *InputWorkPool { count = w.config.InputConcurrency } swg := sizedwaitgroup.New(count) - return &InputWorkPool{Waitgroup: &swg} + return &InputWorkPool{WaitGroup: &swg} } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index c0dbd186b..daa381351 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -62,7 +62,7 @@ type Matcher struct { // description: | // Words contains word patterns required to be present in the response part. // examples: - // - name: Match for outlook mail protection domain + // - name: Match for Outlook mail protection domain // value: > // []string{"mail.protection.outlook.com"} // - name: Match for application/json in response headers diff --git a/v2/pkg/projectfile/httputil.go b/v2/pkg/projectfile/httputil.go index 6479fd20b..6be0b40db 100644 --- a/v2/pkg/projectfile/httputil.go +++ b/v2/pkg/projectfile/httputil.go @@ -79,23 +79,6 @@ func newInternalResponse() *InternalResponse { } } -// Unused -// func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest { -// intReq := newInternalRquest() - -// intReq.Target = target -// intReq.HTTPMajor = req.ProtoMajor -// intReq.HTTPMinor = req.ProtoMinor -// for k, v := range req.Header { -// intReq.Headers[k] = v -// } -// intReq.Headers = req.Header -// intReq.Method = req.Method -// intReq.Body = body - -// return intReq -// } - func toInternalResponse(resp *http.Response, body []byte) *InternalResponse { intResp := newInternalResponse() @@ -125,14 +108,3 @@ func fromInternalResponse(intResp *InternalResponse) *http.Response { Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)), } } - -// Unused -// func fromInternalRequest(intReq *InternalRequest) *http.Request { -// return &http.Request{ -// ProtoMinor: intReq.HTTPMinor, -// ProtoMajor: intReq.HTTPMajor, -// Header: intReq.Headers, -// ContentLength: int64(len(intReq.Body)), -// Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)), -// } -// } diff --git a/v2/pkg/projectfile/project.go b/v2/pkg/projectfile/project.go index 71ad9e365..a64d7db1b 100644 --- a/v2/pkg/projectfile/project.go +++ b/v2/pkg/projectfile/project.go @@ -42,13 +42,13 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) { return nil, fmt.Errorf("not found") } - var httprecord HTTPRecord - httprecord.Response = newInternalResponse() - if err := unmarshal(data, &httprecord); err != nil { + var httpRecord HTTPRecord + httpRecord.Response = newInternalResponse() + if err := unmarshal(data, &httpRecord); err != nil { return nil, err } - return fromInternalResponse(httprecord.Response), nil + return fromInternalResponse(httpRecord.Response), nil } func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error { @@ -57,10 +57,10 @@ func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error { return err } - var httprecord HTTPRecord - httprecord.Request = req - httprecord.Response = toInternalResponse(resp, data) - data, err = marshal(httprecord) + var httpRecord HTTPRecord + httpRecord.Request = req + httpRecord.Response = toInternalResponse(resp, data) + data, err = marshal(httpRecord) if err != nil { return err } diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index 769026907..3fc3bdea7 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -13,12 +13,12 @@ type AttackType int // Supported values for the ProtocolType const ( - // BatteringRamAttack replaces same payload into all of the defined payload positions at once. + // BatteringRamAttack replaces same payload in all the defined payload positions at once BatteringRamAttack AttackType = iota + 1 // PitchForkAttack replaces variables with positional value from multiple wordlists PitchForkAttack - // ClusterbombAttack replaces variables with all possible combinations of values - ClusterbombAttack + // ClusterBombAttack replaces variables with all possible combinations of values + ClusterBombAttack limit ) @@ -26,7 +26,7 @@ const ( var attackTypeMappings = map[AttackType]string{ BatteringRamAttack: "batteringram", PitchForkAttack: "pitchfork", - ClusterbombAttack: "clusterbomb", + ClusterBombAttack: "clusterbomb", } func GetSupportedAttackTypes() []AttackType { diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 07b3a3d0a..ca8f9b655 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -111,7 +111,7 @@ func (i *Iterator) Total() int { count = len(p.values) } } - case ClusterbombAttack: + case ClusterBombAttack: count = 1 for _, p := range i.payloads { count *= len(p.values) @@ -127,7 +127,7 @@ func (i *Iterator) Value() (map[string]interface{}, bool) { return i.batteringRamValue() case PitchForkAttack: return i.pitchforkValue() - case ClusterbombAttack: + case ClusterBombAttack: return i.clusterbombValue() default: return i.batteringRamValue() @@ -184,7 +184,7 @@ func (i *Iterator) clusterbombValue() (map[string]interface{}, bool) { signalNext = false } if !p.next() { - // No more inputs in this inputprovider + // No more inputs in this input provider if index == i.msbIterator { // Reset all previous wordlists and increment the msb counter i.msbIterator++ diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 71a033e6c..24a9e2731 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -54,7 +54,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := catalog.New("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterbombAttack, "", catalogInstance) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 294ddca3a..305cae092 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -13,15 +13,15 @@ import ( // validate validates the payloads if any. func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePath string) error { for name, payload := range payloads { - switch pt := payload.(type) { + switch payloadType := payload.(type) { case string: // check if it's a multiline string list - if len(strings.Split(pt, "\n")) != 1 { + if len(strings.Split(payloadType, "\n")) != 1 { return errors.New("invalid number of lines in payload") } // check if it's a worldlist file and try to load it - if fileExists(pt) { + if fileExists(payloadType) { continue } @@ -29,18 +29,18 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat pathTokens := strings.Split(templatePath, string(os.PathSeparator)) for i := range pathTokens { - tpath := filepath.Join(filepath.Join(pathTokens[:i]...), pt) - if fileExists(tpath) { - payloads[name] = tpath + payloadPath := filepath.Join(filepath.Join(pathTokens[:i]...), payloadType) + if fileExists(payloadPath) { + payloads[name] = payloadPath changed = true break } } if !changed { - return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) + return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", payloadType, name) } case interface{}: - loadedPayloads := types.ToStringSlice(pt) + loadedPayloads := types.ToStringSlice(payloadType) if len(loadedPayloads) == 0 { return fmt.Errorf("the payload %s does not contain enough elements", name) } diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index fa13bd82e..a366a65bd 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -11,15 +11,15 @@ func TestCacheCheckMarkFailed(t *testing.T) { cache.MarkFailed("http://example.com:80") if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil { - require.Equal(t, 1, value, "could not get correct markfailed") + require.Equal(t, 1, value, "could not get correct number of marked failed hosts") } cache.MarkFailed("example.com:80") if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil { - require.Equal(t, 2, value, "could not get correct markfailed") + require.Equal(t, 2, value, "could not get correct number of marked failed hosts") } cache.MarkFailed("example.com") if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil { - require.Equal(t, 1, value, "could not get correct markfailed") + require.Equal(t, 1, value, "could not get correct number of marked failed hosts") } for i := 0; i < 3; i++ { cache.MarkFailed("test") diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 46308c9fe..d603621a9 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -23,7 +23,7 @@ var defaultResolvers = []string{ "8.8.4.4:53", // Google } -// Init initializes the clientpool implementation +// Init initializes the client pool implementation func Init(options *types.Options) error { // Don't create clients if already created in the past. if normalClient != nil { diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 0c1ff929c..f8b445fd6 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -76,7 +76,7 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (in } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, tracedata *retryabledns.TraceData) output.InternalEvent { +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string, traceData *retryabledns.TraceData) output.InternalEvent { return output.InternalEvent{ "host": host, "matched": matched, @@ -91,7 +91,7 @@ func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched strin "template-info": request.options.TemplateInfo, "template-path": request.options.TemplatePath, "type": request.Type().String(), - "trace": traceToString(tracedata, false), + "trace": traceToString(traceData, false), } } @@ -133,10 +133,10 @@ func questionToString(resourceRecords []dns.Question) string { return buffer.String() } -func traceToString(tracedata *retryabledns.TraceData, withSteps bool) string { +func traceToString(traceData *retryabledns.TraceData, withSteps bool) string { buffer := &bytes.Buffer{} - if tracedata != nil { - for i, dnsRecord := range tracedata.DNSData { + if traceData != nil { + for i, dnsRecord := range traceData.DNSData { if withSteps { buffer.WriteString(fmt.Sprintf("request %d to resolver %s:\n", i, strings.Join(dnsRecord.Resolver, ","))) } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index a1a7b5c37..1b6516b85 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -74,15 +74,15 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review gologger.Verbose().Msgf("[%s] Sent DNS request to %s\n", request.options.TemplateID, domain) // perform trace if necessary - var tracedata *retryabledns.TraceData + var traceData *retryabledns.TraceData if request.Trace { - tracedata, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) + traceData, err = request.dnsClient.Trace(domain, request.question, request.TraceMaxRecursion) if err != nil { request.options.Output.Request(request.options.TemplatePath, domain, "dns", err) } } - outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, tracedata) + outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, traceData) for k, v := range previous { outputEvent[k] = v } @@ -92,7 +92,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review dumpResponse(event, request.options, response.String(), domain) if request.Trace { - dumpTraceData(event, request.options, traceToString(tracedata, true), domain) + dumpTraceData(event, request.options, traceToString(traceData, true), domain) } callback(event) @@ -112,15 +112,15 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols. } } -func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, tracedata, domain string) { +func dumpTraceData(event *output.InternalWrappedEvent, requestOptions *protocols.ExecuterOptions, traceData, domain string) { cliOptions := requestOptions.Options if cliOptions.Debug || cliOptions.DebugResponse { hexDump := false - if responsehighlighter.HasBinaryContent(tracedata) { + if responsehighlighter.HasBinaryContent(traceData) { hexDump = true - tracedata = hex.Dump([]byte(tracedata)) + traceData = hex.Dump([]byte(traceData)) } - highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, tracedata, cliOptions.NoColor, hexDump) + highlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, traceData, cliOptions.NoColor, hexDump) gologger.Debug().Msgf("[%s] Dumped DNS Trace data for %s\n\n%s", requestOptions.TemplateID, domain, highlightedResponse) } } diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index c435bef40..d73e79044 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -21,7 +21,7 @@ import ( type Browser struct { customAgent string tempDir string - previouspids map[int32]struct{} // track already running pids + previousPIDs map[int32]struct{} // track already running PIDs engine *rod.Browser httpclient *http.Client options *types.Options @@ -33,7 +33,7 @@ func New(options *types.Options) (*Browser, error) { if err != nil { return nil, errors.Wrap(err, "could not create temporary directory") } - previouspids := findChromeProcesses() + previousPIDs := findChromeProcesses() chromeLauncher := launcher.New(). Leakless(false). @@ -89,7 +89,7 @@ func New(options *types.Options) (*Browser, error) { customAgent = uarand.GetRandom() } - httpclient, err := newhttpClient(options) + httpclient, err := newHttpClient(options) if err != nil { return nil, err } @@ -101,7 +101,7 @@ func New(options *types.Options) (*Browser, error) { httpclient: httpclient, options: options, } - engine.previouspids = previouspids + engine.previousPIDs = previousPIDs return engine, nil } @@ -123,7 +123,7 @@ func (b *Browser) killChromeProcesses() { continue } // skip chrome processes that were already running - if _, ok := b.previouspids[process.Pid]; ok { + if _, ok := b.previousPIDs[process.Pid]; ok { continue } _ = process.Kill() diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index e61803d4b..9b1c5b0df 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -18,8 +18,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// newhttpClient creates a new http client for headless communication with a timeout -func newhttpClient(options *types.Options) (*http.Client, error) { +// newHttpClient creates a new http client for headless communication with a timeout +func newHttpClient(options *types.Options) (*http.Client, error) { dialer := protocolstate.Dialer // Set the base TLS configuration definition diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 234d726eb..b186dcc5a 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -399,12 +399,12 @@ func (p *Page) SelectInputElement(act *Action, out map[string]string /*TODO revi return errors.Wrap(err, "could not scroll into view") } - selectedbool := false + selectedBool := false if act.GetArg("selected") == "true" { - selectedbool = true + selectedBool = true } by := act.GetArg("selector") - if err := element.Select([]string{value}, selectedbool, selectorBy(by)); err != nil { + if err := element.Select([]string{value}, selectedBool, selectorBy(by)); err != nil { return errors.Wrap(err, "could not select input") } return nil @@ -509,7 +509,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused protoEvent := &protoEvent{event: event} // Uses another instance in order to be able to chain the timeout only to the wait operation - pagec := p.page + pageCopy := p.page timeout := act.GetArg("timeout") if timeout != "" { ts, err := strconv.Atoi(timeout) @@ -517,11 +517,11 @@ func (p *Page) WaitEvent(act *Action, out map[string]string /*TODO review unused return errors.Wrap(err, "could not get timeout") } if ts > 0 { - pagec = p.page.Timeout(time.Duration(ts) * time.Second) + pageCopy = p.page.Timeout(time.Duration(ts) * time.Second) } } // Just wait the event to happen - pagec.WaitEvent(protoEvent)() + pageCopy.WaitEvent(protoEvent)() return nil } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 0c5359415..64e9f4da3 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,7 +8,7 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { - // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + // usually browsers don't use chunked transfer encoding, so we set the content-length nevertheless ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) for _, rule := range p.rules { diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 4719a5d24..7cca368a6 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -136,7 +136,7 @@ func TestMakeRequestFromRawWithPayloads(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -175,7 +175,7 @@ func TestMakeRequestFromRawPayloadExpressions(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) @@ -227,7 +227,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) { ServerURL: options.InteractshURL, CacheSize: int64(options.InteractionsCacheSize), Eviction: time.Duration(options.InteractionsEviction) * time.Second, - ColldownPeriod: time.Duration(options.InteractionsCooldownPeriod) * time.Second, + ColldownPeriod: time.Duration(options.InteractionsCoolDownPeriod) * time.Second, PollDuration: time.Duration(options.InteractionsPollDuration) * time.Second, }) require.Nil(t, err, "could not create interactsh client") diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 17a87588d..a96177862 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -41,12 +41,12 @@ type Request struct { // Name is the optional name of the request. // // If a name is specified, all the named request in a template can be matched upon - // in a combined manner allowing multirequest based matchers. + // in a combined manner allowing multi-request based matchers. Name string `yaml:"name,omitempty" jsonschema:"title=name for the http request,description=Optional name for the HTTP Request"` // description: | // Attack is the type of payload combinations to perform. // - // batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates + // batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: // - "batteringram" @@ -137,7 +137,7 @@ type Request struct { dynamicValues map[string]interface{} // description: | - // SelfContained specifies if the request is self contained. + // SelfContained specifies if the request is self-contained. SelfContained bool `yaml:"-" json:"-"` // description: | diff --git a/v2/pkg/protocols/http/http_test.go b/v2/pkg/protocols/http/http_test.go index 4da40ce6f..aca703e9f 100644 --- a/v2/pkg/protocols/http/http_test.go +++ b/v2/pkg/protocols/http/http_test.go @@ -23,7 +23,7 @@ func TestHTTPCompile(t *testing.T) { "username": []string{"admin"}, "password": []string{"admin", "guest", "password", "test", "12345", "123456"}, }, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /manager/html HTTP/1.1 Host: {{Hostname}} User-Agent: Nuclei - Open-source project (github.com/projectdiscovery/nuclei) diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 9305870a6..5859e10cd 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -30,7 +30,7 @@ var ( // Dialer is a copy of the fastdialer from protocolstate Dialer *fastdialer.Dialer - rawhttpClient *rawhttp.Client + rawHttpClient *rawhttp.Client poolMutex *sync.RWMutex normalClient *retryablehttp.Client clientPool map[string]*retryablehttp.Client @@ -98,12 +98,12 @@ func (c *Configuration) HasStandardOptions() bool { // GetRawHTTP returns the rawhttp request client func GetRawHTTP(options *types.Options) *rawhttp.Client { - if rawhttpClient == nil { - rawhttpOptions := rawhttp.DefaultOptions - rawhttpOptions.Timeout = time.Duration(options.Timeout) * time.Second - rawhttpClient = rawhttp.NewClient(rawhttpOptions) + if rawHttpClient == nil { + rawHttpOptions := rawhttp.DefaultOptions + rawHttpOptions.Timeout = time.Duration(options.Timeout) * time.Second + rawHttpClient = rawhttp.NewClient(rawHttpOptions) } - return rawhttpClient + return rawHttpClient } // Get creates or gets a client for the protocol based on custom configuration @@ -138,7 +138,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // Multiple Host - retryablehttpOptions := retryablehttp.DefaultOptionsSpraying + retryableHttpOptions := retryablehttp.DefaultOptionsSpraying disableKeepAlives := true maxIdleConns := 0 maxConnsPerHost := 0 @@ -146,14 +146,14 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl if configuration.Threads > 0 { // Single host - retryablehttpOptions = retryablehttp.DefaultOptionsSingle + retryableHttpOptions = retryablehttp.DefaultOptionsSingle disableKeepAlives = false maxIdleConnsPerHost = 500 maxConnsPerHost = 500 } - retryablehttpOptions.RetryWaitMax = 10 * time.Second - retryablehttpOptions.RetryMax = options.Retries + retryableHttpOptions.RetryWaitMax = 10 * time.Second + retryableHttpOptions.RetryMax = options.Retries followRedirects := configuration.FollowRedirects maxRedirects := configuration.MaxRedirects @@ -213,7 +213,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl Transport: transport, Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), - }, retryablehttpOptions) + }, retryableHttpOptions) if jar != nil { client.HTTPClient.Jar = jar } diff --git a/v2/pkg/protocols/http/race/syncedreadcloser.go b/v2/pkg/protocols/http/race/syncedreadcloser.go index c8f5de7ea..160f64ced 100644 --- a/v2/pkg/protocols/http/race/syncedreadcloser.go +++ b/v2/pkg/protocols/http/race/syncedreadcloser.go @@ -13,7 +13,7 @@ type SyncedReadCloser struct { data []byte p int64 length int64 - opengate chan struct{} + openGate chan struct{} enableBlocking bool } @@ -29,7 +29,7 @@ func NewSyncedReadCloser(r io.ReadCloser) *SyncedReadCloser { } r.Close() s.length = int64(len(s.data)) - s.opengate = make(chan struct{}) + s.openGate = make(chan struct{}) s.enableBlocking = true return &s } @@ -48,13 +48,13 @@ func (s *SyncedReadCloser) SetOpenGate(status bool) { // OpenGate opens the gate allowing all requests to be completed func (s *SyncedReadCloser) OpenGate() { - s.opengate <- struct{}{} + s.openGate <- struct{}{} } // OpenGateAfter schedules gate to be opened after a duration func (s *SyncedReadCloser) OpenGateAfter(d time.Duration) { time.AfterFunc(d, func() { - s.opengate <- struct{}{} + s.openGate <- struct{}{} }) } @@ -84,7 +84,7 @@ func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) { func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { // If the data fits in the buffer blocks awaiting the sync instruction if s.p+int64(len(p)) >= s.length && s.enableBlocking { - <-s.opengate + <-s.openGate } n = copy(p, s.data[s.p:]) s.p += int64(n) @@ -94,7 +94,7 @@ func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { return n, err } -// Close implements close method for io.ReadSeeker +// Close closes an io.ReadSeeker func (s *SyncedReadCloser) Close() error { return nil } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index df4ff5b65..60db9b387 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -57,7 +57,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { // Set the request Method rawRequest.Method = parts[0] - var mutlipartRequest bool + var multiPartRequest bool // Accepts all malformed headers var key, value string for { @@ -76,7 +76,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { value = p[1] } if strings.Contains(key, "Content-Type") && strings.Contains(value, "multipart/") { - mutlipartRequest = true + multiPartRequest = true } // in case of unsafe requests multiple headers should be accepted @@ -136,7 +136,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { return nil, fmt.Errorf("could not read request body: %w", err) } rawRequest.Data = string(b) - if !mutlipartRequest { + if !multiPartRequest { rawRequest.Data = strings.TrimSuffix(rawRequest.Data, "\r\n") } return rawRequest, nil diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..bf92315d7 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -232,7 +232,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou reqURL = generatedHttpRequest.URL() } request.dynamicValues = generatedHttpRequest.dynamicValues - // Check if hosts just keep erroring + // Check if hosts keep erroring if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break } @@ -269,7 +269,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou requestCount++ request.options.Progress.IncrementRequests() - // If this was a match and we want to stop at first match, skip all further requests. + // If this was a match, and we want to stop at first match, skip all further requests. if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } @@ -301,7 +301,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } dumpedRequestString := string(dumpedRequest) - // Check if are there any unresolved variables. If yes, skip unless overriden by user. + // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) return errStopExecution diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index ee8c524c0..8fb0fe70b 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -30,7 +30,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") @@ -54,7 +54,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { req := &Request{ Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, - AttackType: generators.AttackTypeHolder{Value: generators.ClusterbombAttack}, + AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack}, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := catalog.New("") diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 834cc091d..17cbd7ffe 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -98,13 +98,13 @@ func normalizeResponseBody(resp *http.Response, response *redirectedResponse) er // gb18030 supersedes gb2312 responseContentType := resp.Header.Get("Content-Type") if isContentTypeGbk(responseContentType) { - response.fullResponse, err = decodegbk(response.fullResponse) + response.fullResponse, err = decodeGBK(response.fullResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") } // the uncompressed body needs to be decoded to standard utf8 - response.body, err = decodegbk(response.body) + response.body, err = decodeGBK(response.body) if err != nil { return errors.Wrap(err, "could not gbk decode") } @@ -150,8 +150,8 @@ func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte, return bodyDec, nil } -// decodegbk converts GBK to UTF-8 -func decodegbk(s []byte) ([]byte, error) { +// decodeGBK converts GBK to UTF-8 +func decodeGBK(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) d, e := ioutil.ReadAll(O) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 09dcfb451..554d3bea8 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -34,7 +34,7 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates + // Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: // - "batteringram" @@ -68,7 +68,7 @@ type Request struct { ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // description: | - // SelfContained specifies if the request is self contained. + // SelfContained specifies if the request is self-contained. SelfContained bool `yaml:"-" json:"-"` // Operators for the current request go here. @@ -110,7 +110,7 @@ type Input struct { // Read is the number of bytes to read from socket. // // This can be used for protocols which expect an immediate response. You can - // read and write responses one after another and evetually perform matching + // read and write responses one after another and eventually perform matching // on every data captured with `name` attribute. // // The [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this. diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 53a35df3c..d7641dd23 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -300,7 +300,7 @@ func (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map requestOptions.Progress.IncrementFailedRequestsBy(1) return nil, "", errors.Wrap(err, "could not write request to server") } - // Only perform matching and writes in case we recieve + // Only perform matching and writes in case we receive // text or binary opcode from the websocket server. if opCode != ws.OpText && opCode != ws.OpBinary { continue diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index 6e009f42b..a79e80124 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -16,7 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" ) -// Options contains necessary options required for elasticsearch communicaiton +// Options contains necessary options required for elasticsearch communication type Options struct { // IP for elasticsearch instance IP string `yaml:"ip" validate:"required,ip"` diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index d8f000584..f51470e1c 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -24,10 +24,10 @@ type Options struct { AllowList *Filter `yaml:"allow-list"` // DenyList contains a list of denied events for reporting module DenyList *Filter `yaml:"deny-list"` - // Github contains configuration options for Github Issue Tracker - Github *github.Options `yaml:"github"` - // Gitlab contains configuration options for Gitlab Issue Tracker - Gitlab *gitlab.Options `yaml:"gitlab"` + // GitHub contains configuration options for GitHub Issue Tracker + GitHub *github.Options `yaml:"github"` + // GitLab contains configuration options for GitLab Issue Tracker + GitLab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` // MarkdownExporter contains configuration options for Markdown Exporter Module @@ -107,15 +107,15 @@ type Client struct { // New creates a new nuclei issue tracker reporting client func New(options *Options, db string) (*Client, error) { client := &Client{options: options} - if options.Github != nil { - tracker, err := github.New(options.Github) + if options.GitHub != nil { + tracker, err := github.New(options.GitHub) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") } client.trackers = append(client.trackers, tracker) } - if options.Gitlab != nil { - tracker, err := gitlab.New(options.Gitlab) + if options.GitLab != nil { + tracker, err := gitlab.New(options.GitLab) if err != nil { return nil, errors.Wrap(err, "could not create reporting client") } diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 47225ada1..352c5183f 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -21,15 +21,15 @@ type Integration struct { options *Options } -// Options contains the configuration options for github issue tracker client +// Options contains the configuration options for GitHub issue tracker client type Options struct { - // BaseURL (optional) is the self-hosted github application url + // BaseURL (optional) is the self-hosted GitHub application url BaseURL string `yaml:"base-url" validate:"omitempty,url"` // Username is the username of the github user Username string `yaml:"username" validate:"required"` - // Owner (manadatory) is the owner name of the repository for issues. + // Owner is the owner name of the repository for issues. Owner string `yaml:"owner" validate:"required"` - // Token is the token for github account. + // Token is the token for GitHub account. Token string `yaml:"token" validate:"required"` // ProjectName is the name of the repository. ProjectName string `yaml:"project-name" validate:"required"` diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index eb547d0a6..088eab1b8 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -331,7 +331,7 @@ func init() { HTTPRequestDoc.Fields[6].Name = "name" HTTPRequestDoc.Fields[6].Type = "string" HTTPRequestDoc.Fields[6].Note = "" - HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multirequest based matchers." + HTTPRequestDoc.Fields[6].Description = "Name is the optional name of the request.\n\nIf a name is specified, all the named request in a template can be matched upon\nin a combined manner allowing multi-request based matchers." HTTPRequestDoc.Fields[6].Comments[encoder.LineComment] = "Name is the optional name of the request." HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 51fec82fd..99ddf9cef 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -61,7 +61,7 @@ var DefaultOptions = &types.Options{ InteractshURL: "https://interactsh.com", InteractionsCacheSize: 5000, InteractionsEviction: 60, - InteractionsCooldownPeriod: 5, + InteractionsCoolDownPeriod: 5, InteractionsPollDuration: 5, } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 1a0935338..69a26f5e0 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -70,7 +70,7 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string - // MarkdownExportDirectory is the directory to export reports in markdown format + // MarkdownExportDirectory is the directory to export reports in Markdown format MarkdownExportDirectory string // SarifExport is the file to export sarif output format to SarifExport string @@ -107,9 +107,9 @@ type Options struct { // Eviction is the number of seconds after which to automatically discard // interaction requests. InteractionsEviction int - // InteractionsCooldownPeriod is additional seconds to wait for interactions after closing + // InteractionsCoolDownPeriod is additional seconds to wait for interactions after closing // of the poller. - InteractionsCooldownPeriod int + InteractionsCoolDownPeriod int // OfflineHTTP is a flag that specific offline processing of http response // using same matchers/extractors from http protocol without the need // to send a new request, reading responses from a file. diff --git a/v2/pkg/utils/stats/stats.go b/v2/pkg/utils/stats/stats.go index d25d14334..9b2b7f2f0 100644 --- a/v2/pkg/utils/stats/stats.go +++ b/v2/pkg/utils/stats/stats.go @@ -30,7 +30,7 @@ func NewEntry(name, description string) { Default.NewEntry(name, description) } -// Increment incrmements the value for a name string +// Increment increments the value for a name string func Increment(name string) { Default.Increment(name) } @@ -57,7 +57,7 @@ func (s *Storage) NewEntry(name, description string) { s.mutex.Unlock() } -// Increment incrmements the value for a name string +// Increment increments the value for a name string func (s *Storage) Increment(name string) { s.mutex.RLock() data, ok := s.data[name] From 5eca474c5e227db149865a3d4619d11914b30680 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 26 Nov 2021 11:38:09 +0200 Subject: [PATCH 165/196] refactor(CR): better way of writing switch statement --- v2/pkg/protocols/headless/engine/rules.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index 64e9f4da3..8dc206876 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -16,16 +16,16 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - switch { - case rule.Action == ActionSetMethod: + switch rule.Action { + case ActionSetMethod: ctx.Request.Req().Method = rule.Args["method"] - case rule.Action == ActionAddHeader: + case ActionAddHeader: ctx.Request.Req().Header.Add(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionSetHeader: + case ActionSetHeader: ctx.Request.Req().Header.Set(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionDeleteHeader: + case ActionDeleteHeader: ctx.Request.Req().Header.Del(rule.Args["key"]) - case rule.Action == ActionSetBody: + case ActionSetBody: body := rule.Args["body"] ctx.Request.Req().ContentLength = int64(len(body)) ctx.Request.SetBody(body) @@ -38,14 +38,14 @@ func (p *Page) routingRuleHandler(ctx *rod.Hijack) { continue } - switch { - case rule.Action == ActionAddHeader: + switch rule.Action { + case ActionAddHeader: ctx.Response.Headers().Add(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionSetHeader: + case ActionSetHeader: ctx.Response.Headers().Set(rule.Args["key"], rule.Args["value"]) - case rule.Action == ActionDeleteHeader: + case ActionDeleteHeader: ctx.Response.Headers().Del(rule.Args["key"]) - case rule.Action == ActionSetBody: + case ActionSetBody: body := rule.Args["body"] ctx.Response.Headers().Set("Content-Length", fmt.Sprintf("%d", len(body))) ctx.Response.SetBody(rule.Args["body"]) From c720354be23663078847b4a787be41cf31f18c90 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 26 Nov 2021 16:23:54 +0530 Subject: [PATCH 166/196] Added part definition information to docs + misc --- SYNTAX-REFERENCE.md | 131 ++++++++++ v2/go.mod | 2 +- v2/go.sum | 2 + v2/pkg/protocols/dns/dns.go | 20 ++ v2/pkg/protocols/file/file.go | 13 + v2/pkg/protocols/headless/headless.go | 14 + v2/pkg/protocols/http/http.go | 22 ++ v2/pkg/protocols/network/network.go | 15 ++ v2/pkg/protocols/offlinehttp/offlinehttp.go | 22 ++ v2/pkg/protocols/ssl/ssl.go | 11 + v2/pkg/protocols/websocket/websocket.go | 12 + v2/pkg/templates/templates_doc.go | 274 ++++++++++++++++++++ 12 files changed, 537 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index c69801fbd..42f2c5cdf 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -9,6 +9,8 @@ Template is a YAML input file which defines all the requests and + +

@@ -290,6 +292,8 @@ reference: https://zxsecurity.co.nz/research/argunment-injection-ruby-dragonfly/ severity: high ``` + +
@@ -545,6 +549,8 @@ CWE-22 + + ## severity.Holder Holder holds a Severity type. Required for un/marshalling purposes @@ -555,6 +561,8 @@ Appears in: + +
@@ -598,6 +606,8 @@ Appears in: + +
@@ -724,6 +734,26 @@ path: method: GET ``` +Part Definitions: + + +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - HTTP request made from the client +- response - HTTP response recieved from server +- status_code - Status Code received from the Server +- body - HTTP response body received from server (default) +- content_length - HTTP Response content length +- header,all_headers - HTTP response headers +- duration - HTTP request time duration +- all - HTTP response body + headers +- - HTTP response cookies in : format +- - HTTP response headers in : format +
@@ -1266,6 +1296,8 @@ Appears in: + +
@@ -1599,6 +1631,8 @@ Appears in: + +
@@ -1655,6 +1689,8 @@ Appears in: + +
@@ -1935,6 +1971,8 @@ Appears in: + +
@@ -1979,6 +2017,8 @@ Appears in: + +
@@ -2017,6 +2057,8 @@ Appears in: + +
@@ -2081,6 +2123,24 @@ retries: 2 recursion: true ``` +Part Definitions: + + +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- request - Request contains the DNS request in text format +- type - Type is the type of request made +- rcode - Rcode field returned for the DNS request +- question - Question contains the DNS question field +- extra - Extra contains the DNS response extra field +- answer - Answer contains the DNS response answer field +- ns - NS contains the DNS response NS field +- raw,body,all - Raw contains the raw DNS response (default) +- trace - Trace contains trace data for DNS request if enabled +
@@ -2317,6 +2377,8 @@ Appears in: + +
@@ -2375,6 +2437,17 @@ extensions: - all ``` +Part Definitions: + + +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- matched - Matched is the input which was matched upon +- path - Path is the path of file on local filesystem +- type - Type is the type of request made +- raw,body,all,data - Raw contains the raw file contents +
@@ -2562,6 +2635,19 @@ matchers: - zookeeper.version ``` +Part Definitions: + + +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - Network request made from the client +- body,all,data - Network response recieved from server (default) +- raw - Full Network protocol data +
@@ -2765,6 +2851,8 @@ Appears in: + +
@@ -2884,6 +2972,8 @@ Appears in: + +
@@ -2920,6 +3010,18 @@ Appears in: +Part Definitions: + + +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- req - Headless request made from the client +- resp,body,data - Headless response recieved from client (default) +
@@ -3019,6 +3121,8 @@ Appears in: + +
@@ -3092,6 +3196,8 @@ Appears in: + +
@@ -3168,6 +3274,15 @@ Appears in: +Part Definitions: + + +- type - Type is the type of request made +- response - JSON SSL protocol handshake details +- not_after - Timestamp after which the remote cert expires +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
@@ -3249,6 +3364,16 @@ Appears in: +Part Definitions: + + +- type - Type is the type of request made +- success - Success specifies whether websocket connection was successful +- request - Websocket request made to the server +- response - Websocket response recieved from the server +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
@@ -3388,6 +3513,8 @@ Appears in: + +
@@ -3459,6 +3586,8 @@ Appears in: + +
@@ -3542,6 +3671,8 @@ Appears in: + +
diff --git a/v2/go.mod b/v2/go.mod index cbed75130..709577509 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -40,7 +40,7 @@ require ( github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 - github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e + github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.4 diff --git a/v2/go.sum b/v2/go.sum index 1f385e95e..731a08159 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -656,6 +656,8 @@ github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0 h1:m github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e h1:0ZxOM0Q0/ESa24L/vq3fxs9YipxfHR4Y3jM/H2ReJ5E= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= +github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 7b01d1390..b3f3f7c1a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -78,6 +78,26 @@ type Request struct { Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "request": "Request contains the DNS request in text format", + "type": "Type is the type of request made", + "rcode": "Rcode field returned for the DNS request", + "question": "Question contains the DNS question field", + "extra": "Extra contains the DNS response extra field", + "answer": "Answer contains the DNS response answer field", + "ns": "NS contains the DNS response NS field", + "raw,body,all": "Raw contains the raw DNS response (default)", + "trace": "Trace contains trace data for DNS request if enabled", +} + func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index e61965025..b0902e7a0 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -52,6 +52,19 @@ type Request struct { allExtensions bool } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "matched": "Matched is the input which was matched upon", + "path": "Path is the path of file on local filesystem", + "type": "Type is the type of request made", + "raw,body,all,data": "Raw contains the raw file contents", +} + // defaultDenylist is the default list of extensions to be denied var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 43f62f437..c3664faff 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -25,6 +25,20 @@ type Request struct { options *protocols.ExecuterOptions } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "req": "Headless request made from the client", + "resp,body,data": "Headless response recieved from client (default)", +} + // Step is a headless protocol request step. type Step struct { // Action is the headless action to execute for the script diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index fca27c162..6a6f2d49b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -167,6 +167,28 @@ type Request struct { SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "request": "HTTP request made from the client", + "response": "HTTP response recieved from server", + "status_code": "Status Code received from the Server", + "body": "HTTP response body received from server (default)", + "content_length": "HTTP Response content length", + "header,all_headers": "HTTP response headers", + "duration": "HTTP request time duration", + "all": "HTTP response body + headers", + "": "HTTP response cookies in : format", + "": "HTTP response headers in : format", +} + // GetID returns the unique ID of the request if any. func (request *Request) GetID() string { return request.ID diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c943b42e2..c6ab7d712 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -78,6 +78,21 @@ type Request struct { dynamicValues map[string]interface{} } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "request": "Network request made from the client", + "body,all,data": "Network response recieved from server (default)", + "raw": "Full Network protocol data", +} + type addressKV struct { ip string port string diff --git a/v2/pkg/protocols/offlinehttp/offlinehttp.go b/v2/pkg/protocols/offlinehttp/offlinehttp.go index 85b669320..6f65ac426 100644 --- a/v2/pkg/protocols/offlinehttp/offlinehttp.go +++ b/v2/pkg/protocols/offlinehttp/offlinehttp.go @@ -13,6 +13,28 @@ type Request struct { compiledOperators []*operators.Operators } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "request": "HTTP request made from the client", + "response": "HTTP response recieved from server", + "status_code": "Status Code received from the Server", + "body": "HTTP response body received from server (default)", + "content_length": "HTTP Response content length", + "header,all_headers": "HTTP response headers", + "duration": "HTTP request time duration", + "all": "HTTP response body + headers", + "": "HTTP response cookies in : format", + "": "HTTP response headers in : format", +} + // GetID returns the unique ID of the request if any. func (request *Request) GetID() string { return "" diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 388886be2..62403581b 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -145,6 +145,17 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous return nil } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "type": "Type is the type of request made", + "response": "JSON SSL protocol handshake details", + "not_after": "Timestamp after which the remote cert expires", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", +} + // getAddress returns the address of the host to make request to func getAddress(toTest string) (string, error) { if strings.Contains(toTest, "://") { diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index d3026ee6a..e9958afce 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -357,6 +357,18 @@ func (request *Request) GetCompiledOperators() []*operators.Operators { return []*operators.Operators{request.CompiledOperators} } +// RequestPartDefinitions contains a mapping of request part definitions and their +// description. Multiple definitions are separated by commas. +// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. +var RequestPartDefinitions = map[string]string{ + "type": "Type is the type of request made", + "success": "Success specifies whether websocket connection was successful", + "request": "Websocket request made to the server", + "response": "Websocket response recieved from the server", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", +} + func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(request.options.TemplateID), diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index b3ce14f71..bbba306d0 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -301,6 +301,72 @@ func init() { FieldName: "requests", }, } + HTTPRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "HTTP request made from the client", + }, + { + Key: "response", + Value: "HTTP response recieved from server", + }, + { + Key: "status_code", + Value: "Status Code received from the Server", + }, + { + Key: "body", + Value: "HTTP response body received from server (default)", + }, + { + Key: "content_length", + Value: "HTTP Response content length", + }, + { + Key: "header,all_headers", + Value: "HTTP response headers", + }, + { + Key: "duration", + Value: "HTTP request time duration", + }, + { + Key: "all", + Value: "HTTP response body + headers", + }, + { + Key: "", + Value: "HTTP response cookies in : format", + }, + { + Key: "", + Value: "HTTP response headers in : format", + }, + } HTTPRequestDoc.Fields = make([]encoder.Doc, 26) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -827,6 +893,64 @@ func init() { FieldName: "dns", }, } + DNSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "request", + Value: "Request contains the DNS request in text format", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "rcode", + Value: "Rcode field returned for the DNS request", + }, + { + Key: "question", + Value: "Question contains the DNS question field", + }, + { + Key: "extra", + Value: "Extra contains the DNS response extra field", + }, + { + Key: "answer", + Value: "Answer contains the DNS response answer field", + }, + { + Key: "ns", + Value: "NS contains the DNS response NS field", + }, + { + Key: "raw,body,all", + Value: "Raw contains the raw DNS response (default)", + }, + { + Key: "trace", + Value: "Trace contains trace data for DNS request if enabled", + }, + } DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -945,6 +1069,36 @@ func init() { FieldName: "file", }, } + FILERequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "path", + Value: "Path is the path of file on local filesystem", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "raw,body,all,data", + Value: "Raw contains the raw file contents", + }, + } FILERequestDoc.Fields = make([]encoder.Doc, 8) FILERequestDoc.Fields[0].Name = "matchers" FILERequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1008,6 +1162,44 @@ func init() { FieldName: "network", }, } + NETWORKRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "Network request made from the client", + }, + { + Key: "body,all,data", + Value: "Network response recieved from server (default)", + }, + { + Key: "raw", + Value: "Full Network protocol data", + }, + } NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" @@ -1142,6 +1334,40 @@ func init() { FieldName: "headless", }, } + HEADLESSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "req", + Value: "Headless request made from the client", + }, + { + Key: "resp,body,data", + Value: "Headless response recieved from client (default)", + }, + } HEADLESSRequestDoc.Fields = make([]encoder.Doc, 5) HEADLESSRequestDoc.Fields[0].Name = "id" HEADLESSRequestDoc.Fields[0].Type = "string" @@ -1253,6 +1479,28 @@ func init() { FieldName: "ssl", }, } + SSLRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "response", + Value: "JSON SSL protocol handshake details", + }, + { + Key: "not_after", + Value: "Timestamp after which the remote cert expires", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } SSLRequestDoc.Fields = make([]encoder.Doc, 4) SSLRequestDoc.Fields[0].Name = "matchers" SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1288,6 +1536,32 @@ func init() { FieldName: "websocket", }, } + WEBSOCKETRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "success", + Value: "Success specifies whether websocket connection was successful", + }, + { + Key: "request", + Value: "Websocket request made to the server", + }, + { + Key: "response", + Value: "Websocket response recieved from the server", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) WEBSOCKETRequestDoc.Fields[0].Name = "matchers" WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" From dfe284664ce1f334a9a5baf04e925264ab62f219 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 26 Nov 2021 18:51:02 +0530 Subject: [PATCH 167/196] Fixed a crash with http module (#1285) --- v2/pkg/protocols/http/request.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 8f2040782..a0bce62ee 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -405,6 +405,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } var dumpedResponse []redirectedResponse + var gotData []byte // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader @@ -422,6 +423,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return errors.Wrap(err, "could not read http body") } } + gotData = data resp.Body.Close() dumpedResponse, err = dumpResponseWithRedirectChain(resp, data) @@ -432,14 +434,17 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate dumpedResponse = []redirectedResponse{{fullResponse: dumpedResponseHeaders, headers: dumpedResponseHeaders}} } - for _, response := range dumpedResponse { - // if nuclei-project is enabled store the response if not previously done - if request.options.ProjectFile != nil && !fromCache { - if err := request.options.ProjectFile.Set(dumpedRequest, resp, response.body); err != nil { - return errors.Wrap(err, "could not store in project file") - } + // if nuclei-project is enabled store the response if not previously done + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, gotData); err != nil { + return errors.Wrap(err, "could not store in project file") } + } + for _, response := range dumpedResponse { + if response.resp == nil { + continue // Skip nil responses + } matchedURL := reqURL if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { matchedURL = generatedRequest.rawRequest.FullURL From cd651ddf6f6af7ddd4d5fd313c885527305c0bdb Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Fri, 26 Nov 2021 17:12:27 +0200 Subject: [PATCH 168/196] fix: Escaped the dot in a regex matching a domain --- v2/cmd/docgen/docgen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/docgen/docgen.go b/v2/cmd/docgen/docgen.go index 1c8e9779a..7cfdaf275 100644 --- a/v2/cmd/docgen/docgen.go +++ b/v2/cmd/docgen/docgen.go @@ -14,7 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -var pathRegex = regexp.MustCompile(`github.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) +var pathRegex = regexp.MustCompile(`github\.com/projectdiscovery/nuclei/v2/(?:internal|pkg)/(?:.*/)?([A-Za-z.]+)`) func main() { // Generate yaml syntax documentation From b59582bde7188932cd9776c5433e43eff4caba4c Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 28 Nov 2021 04:31:39 +0530 Subject: [PATCH 169/196] Fixed stdin input parsing bug (#1286) --- v2/internal/runner/options.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..c382f69d2 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,9 +71,8 @@ func hasStdin() bool { return false } - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) != 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - return isPipedFromChrDev || isPipedFromFIFO } From b3805999f357843cd138f5fab3719795fb880910 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Sat, 27 Nov 2021 17:10:27 -0600 Subject: [PATCH 170/196] Unsafe flag with base template (#1279) * unsafe flag with base template --- v2/pkg/protocols/http/build_request.go | 6 ++++-- v2/pkg/protocols/http/request.go | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 8c8a2dbc8..be1d13110 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -307,10 +307,12 @@ func (r *requestGenerator) fillRequest(req *http.Request, values map[string]inte } req.Body = ioutil.NopCloser(strings.NewReader(body)) } - setHeader(req, "User-Agent", uarand.GetRandom()) + if !r.request.Unsafe { + setHeader(req, "User-Agent", uarand.GetRandom()) + } // Only set these headers on non-raw requests - if len(r.request.Raw) == 0 { + if len(r.request.Raw) == 0 && !r.request.Unsafe { setHeader(req, "Accept", "*/*") setHeader(req, "Accept-Language", "en") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a0bce62ee..a38b56b9c 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -310,9 +310,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) - } + } } - var formedURL string var hostname string timeStart := time.Now() From 5a1c7a62b2d6041895050c48bc000ab54e3c6ec4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 28 Nov 2021 04:55:18 +0530 Subject: [PATCH 171/196] missing goimports --- v2/pkg/protocols/network/network.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 82eb4aa50..ef03cc92b 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,8 +1,6 @@ package network import ( - "fmt" - "net" "strings" "github.com/pkg/errors" From 75333dae3b5f558eee60f261b96db938a1e5802e Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 28 Nov 2021 05:29:01 +0530 Subject: [PATCH 172/196] Revert "Fixed stdin input parsing bug (#1286)" This reverts commit b59582bde7188932cd9776c5433e43eff4caba4c. --- v2/internal/runner/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index c382f69d2..f11d2055c 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -71,8 +71,9 @@ func hasStdin() bool { return false } - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) != 0 + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 + return isPipedFromChrDev || isPipedFromFIFO } From b1a0da290a034f413ee614ed0e83089beecb1e2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 10:52:29 +0530 Subject: [PATCH 173/196] chore(deps): bump alpine from 3.14 to 3.15.0 (#1305) Bumps alpine from 3.14 to 3.15.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c5c46e839..322fd54d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.17.3-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest -FROM alpine:3.14 +FROM alpine:3.15.0 RUN apk add --no-cache bind-tools ca-certificates chromium COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei ENTRYPOINT ["nuclei"] From 345a5a3de5ed305472b3de11656e369f0962bde5 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 29 Nov 2021 06:49:56 +0000 Subject: [PATCH 174/196] Auto Generate Syntax Docs + JSONSchema [Mon Nov 29 06:49:56 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 10 +++++----- v2/pkg/templates/templates_doc.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 6c1742432..be39aa225 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -847,7 +847,7 @@ ID is the optional id of the request Name is the optional name of the request. If a name is specified, all the named request in a template can be matched upon -in a combined manner allowing multirequest based matchers. +in a combined manner allowing multi-request based matchers.
@@ -862,7 +862,7 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates +batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. @@ -1454,7 +1454,7 @@ Examples: ```yaml -# Match for outlook mail protection domain +# Match for Outlook mail protection domain words: - mail.protection.outlook.com ``` @@ -2452,7 +2452,7 @@ host: Attack is the type of payload combinations to perform. -Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. @@ -2676,7 +2676,7 @@ Valid values: Read is the number of bytes to read from socket. This can be used for protocols which expect an immediate response. You can -read and write responses one after another and evetually perform matching +read and write responses one after another and eventually perform matching on every data captured with `name` attribute. The [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this. diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 088eab1b8..b436ef472 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -336,7 +336,7 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "generators.AttackTypeHolder" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ "batteringram", @@ -559,7 +559,7 @@ func init() { MATCHERSMatcherDoc.Fields[7].Description = "Words contains word patterns required to be present in the response part." MATCHERSMatcherDoc.Fields[7].Comments[encoder.LineComment] = "Words contains word patterns required to be present in the response part." - MATCHERSMatcherDoc.Fields[7].AddExample("Match for outlook mail protection domain", []string{"mail.protection.outlook.com"}) + MATCHERSMatcherDoc.Fields[7].AddExample("Match for Outlook mail protection domain", []string{"mail.protection.outlook.com"}) MATCHERSMatcherDoc.Fields[7].AddExample("Match for application/json in response headers", []string{"application/json"}) MATCHERSMatcherDoc.Fields[8].Name = "regex" @@ -938,7 +938,7 @@ func init() { NETWORKRequestDoc.Fields[2].Name = "attack" NETWORKRequestDoc.Fields[2].Type = "generators.AttackTypeHolder" NETWORKRequestDoc.Fields[2].Note = "" - NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." NETWORKRequestDoc.Fields[2].Values = []string{ "batteringram", @@ -1020,7 +1020,7 @@ func init() { NETWORKInputDoc.Fields[2].Name = "read" NETWORKInputDoc.Fields[2].Type = "int" NETWORKInputDoc.Fields[2].Note = "" - NETWORKInputDoc.Fields[2].Description = "Read is the number of bytes to read from socket.\n\nThis can be used for protocols which expect an immediate response. You can\nread and write responses one after another and evetually perform matching\non every data captured with `name` attribute.\n\nThe [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this." + NETWORKInputDoc.Fields[2].Description = "Read is the number of bytes to read from socket.\n\nThis can be used for protocols which expect an immediate response. You can\nread and write responses one after another and eventually perform matching\non every data captured with `name` attribute.\n\nThe [network docs](https://nuclei.projectdiscovery.io/templating-guide/protocols/network/) highlight more on how to do this." NETWORKInputDoc.Fields[2].Comments[encoder.LineComment] = "Read is the number of bytes to read from socket." NETWORKInputDoc.Fields[2].AddExample("", 1024) From 5deb454a81f4e32639d6f2e2ad606ba425499130 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 13:26:03 +0530 Subject: [PATCH 175/196] merge Hostname variable to payloads in network request --- v2/pkg/protocols/network/request.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 484f95ae2..24d8b260d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -78,7 +78,8 @@ func (request *Request) executeAddress(actualAddress, address, input string, sho } payloads := generators.BuildPayloadFromOptions(request.options.Options) - generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + // add Hostname variable to the payload + payloads = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) if request.generator != nil { iterator := request.generator.NewIterator() From c9943c0b2a4f3a9e15d1bdc53dec5c96f68327fc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 29 Nov 2021 14:38:55 +0530 Subject: [PATCH 176/196] fix: #1274 spawned nuclei child process hangs reading stdin Next and final attempt at fixing the stdin parsing issue when spawing as child from nodejs --- v2/internal/runner/options.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..d9eaf3adc 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -66,15 +66,14 @@ func ParseOptions(options *types.Options) { // hasStdin returns true if we have stdin input func hasStdin() bool { - stat, err := os.Stdin.Stat() + fi, err := os.Stdin.Stat() if err != nil { return false } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } + return true } // validateOptions validates the configuration options passed From f60e93aae1b7d578c3d0a169a9cec72de7597901 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 16:01:06 +0530 Subject: [PATCH 177/196] add template level stop at first match option --- v2/pkg/protocols/common/executer/executer.go | 5 +++++ v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/protocols/protocols.go | 2 ++ v2/pkg/templates/compile.go | 1 + v2/pkg/templates/templates.go | 3 +++ 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index fe650e945..e2f653856 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -81,6 +81,11 @@ func (e *Executer) Execute(input string) (bool, error) { } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } + // + if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { + gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) + break + } } return results, nil } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1868f85bc..cfe6daf56 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -270,7 +270,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou request.options.Progress.IncrementRequests() // If this was a match, and we want to stop at first match, skip all further requests. - if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } } @@ -310,7 +310,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", dumpedRequestString) - } + } } var formedURL string var hostname string diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index f288b717f..fb7f04a1b 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -61,6 +61,8 @@ type ExecuterOptions struct { Interactsh *interactsh.Client // HostErrorsCache is an optional cache for handling host errors HostErrorsCache *hosterrorscache.Cache + // Stop execution once first match is found + StopAtFirstMatch bool Operators []*operators.Operators // only used by offlinehttp module diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 2899ffc6b..079c96f4a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -69,6 +69,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplateID = template.ID options.TemplateInfo = template.Info options.TemplatePath = filePath + options.StopAtFirstMatch = template.StopAtFirstMatch // If no requests, and it is also not a workflow, return error. if template.Requests() == 0 { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index a7f272732..0eb400afe 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -74,6 +74,9 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + // description: | + // Stop execution once first match is found + StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop at first match for the template"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` From a7ecadf1892881985567d68528fd8e845fb84ee1 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 29 Nov 2021 18:06:25 +0530 Subject: [PATCH 178/196] add template level stop-at-first-match to workflows --- v2/pkg/protocols/common/executer/executer.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index e2f653856..cfefe55a4 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -81,7 +81,7 @@ func (e *Executer) Execute(input string) (bool, error) { } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } - // + // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break @@ -94,6 +94,7 @@ func (e *Executer) Execute(input string) (bool, error) { func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) previous := make(map[string]interface{}) + var results bool for _, req := range e.requests { req := req @@ -113,6 +114,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve if event.OperatorsResult == nil { return } + results = true callback(event) }) if err != nil { @@ -123,6 +125,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } + // If a match was found and stop at first match is set, break out of the loop and return + if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { + gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) + break + } } return nil } From 65a1858f940797d735bc42a7f3a06f2a2883289d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 30 Nov 2021 00:17:52 +0530 Subject: [PATCH 179/196] fix #1303: speed regression due to missing goroutine in executer Fixed speed regression introduced in dev due to missing go() statement when executing template input --- v2/pkg/core/execute.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 60b6c20bf..9e58a3d3f 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -39,15 +39,17 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp } wg.Add() - switch { - case template.SelfContained: - // Self Contained requests are executed here separately - e.executeSelfContainedTemplateWithInput(template, results) - default: - // All other request types are executed here - e.executeModelWithInput(templateType, template, target, results) - } - wg.Done() + go func(tpl *templates.Template) { + switch { + case tpl.SelfContained: + // Self Contained requests are executed here separately + e.executeSelfContainedTemplateWithInput(tpl, results) + default: + // All other request types are executed here + e.executeModelWithInput(templateType, tpl, target, results) + } + wg.Done() + }(template) } e.workPool.Wait() return results From 684f332599f674210512003e3bdef545578eb73f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 30 Nov 2021 12:04:47 +0530 Subject: [PATCH 180/196] fix: {{Hostname}} to {{Host}} in test --- v2/pkg/protocols/network/network_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/network_test.go b/v2/pkg/protocols/network/network_test.go index 730f9584a..27a64d592 100644 --- a/v2/pkg/protocols/network/network_test.go +++ b/v2/pkg/protocols/network/network_test.go @@ -17,7 +17,7 @@ func TestNetworkCompileMake(t *testing.T) { templateID := "testing-network" request := &Request{ ID: templateID, - Address: []string{"tls://{{Hostname}}:443"}, + Address: []string{"tls://{{Host}}:443"}, ReadSize: 1024, Inputs: []*Input{{Data: "test-data"}}, } From 847e9d7c4422d1a1c7cfa6602de5c8a9b47bcb06 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 30 Nov 2021 16:26:29 +0530 Subject: [PATCH 181/196] Misc changes to fix testa and lint --- .../model/types/severity/severity_holder.go | 47 ------------------- .../common/generators/attack_types.go | 2 +- 2 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 v2/pkg/model/types/severity/severity_holder.go diff --git a/v2/pkg/model/types/severity/severity_holder.go b/v2/pkg/model/types/severity/severity_holder.go deleted file mode 100644 index 7d52f9c81..000000000 --- a/v2/pkg/model/types/severity/severity_holder.go +++ /dev/null @@ -1,47 +0,0 @@ -package severity - -import ( - "encoding/json" - - "github.com/alecthomas/jsonschema" -) - -// Holder holds a Severity type. Required for un/marshalling purposes -type Holder struct { - Severity Severity -} - -func (severityHolder Holder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ - Type: "string", - Title: "severity of the template", - Description: "Seriousness of the implications of the template", - } - for _, severity := range GetSupportedSeverities() { - gotType.Enum = append(gotType.Enum, severity.String()) - } - return gotType -} - -func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error { - var marshalledSeverity string - if err := unmarshal(&marshalledSeverity); err != nil { - return err - } - - computedSeverity, err := toSeverity(marshalledSeverity) - if err != nil { - return err - } - - severityHolder.Severity = computedSeverity - return nil -} - -func (severityHolder *Holder) MarshalJSON() ([]byte, error) { - return json.Marshal(severityHolder.Severity.String()) -} - -func (severityHolder Holder) MarshalYAML() (interface{}, error) { - return severityHolder.Severity.String(), nil -} diff --git a/v2/pkg/protocols/common/generators/attack_types.go b/v2/pkg/protocols/common/generators/attack_types.go index 09ac96330..a0a9eb78d 100644 --- a/v2/pkg/protocols/common/generators/attack_types.go +++ b/v2/pkg/protocols/common/generators/attack_types.go @@ -19,7 +19,7 @@ const ( // name:pitchfork PitchForkAttack // name:clusterbomb - ClusterbombAttack + ClusterBombAttack limit ) From de236379a1d8d1937bd94c1895c35c27559f8459 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 30 Nov 2021 17:22:39 +0530 Subject: [PATCH 182/196] remove stop-at-first-match info log #1188 --- v2/pkg/protocols/common/executer/executer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index cfefe55a4..16a3ede71 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -83,7 +83,6 @@ func (e *Executer) Execute(input string) (bool, error) { } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break } } @@ -127,7 +126,6 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - gologger.Info().Msgf("[%s] Stopping execution at first match for %s\n", e.options.TemplateID, input) break } } From 5b99921d75b48f30d22f8ead6d459f6e58337a7f Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 30 Nov 2021 20:20:43 +0530 Subject: [PATCH 183/196] add unit and integration tests for dsl variable #555 --- .../http/dsl-matcher-variable.yaml | 23 +++++++++++++++++++ v2/cmd/integration-test/http.go | 22 ++++++++++++++++++ v2/pkg/operators/matchers/match_test.go | 18 +++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 integration_tests/http/dsl-matcher-variable.yaml diff --git a/integration_tests/http/dsl-matcher-variable.yaml b/integration_tests/http/dsl-matcher-variable.yaml new file mode 100644 index 000000000..ecbe5f9e5 --- /dev/null +++ b/integration_tests/http/dsl-matcher-variable.yaml @@ -0,0 +1,23 @@ +id: dsl-matcher-variable + +info: + name: dsl-matcher-variable + author: pd-team + severity: info + +requests: + - + path: + - "{{BaseURL}}" + payloads: + VALUES: + - This + - is + - test + - matcher + - text + matchers: + - + dsl: + - 'contains(body,"{{VALUES}}")' + type: dsl \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 1890048d2..ebf245317 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -36,6 +36,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, + "http/dsl-matcher-variable.yaml": &httpDSLVariable{}, } type httpInteractshRequest struct{} @@ -155,6 +156,27 @@ func (h *httpGet) Execute(filePath string) error { return nil } +type httpDSLVariable struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpDSLVariable) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "This is test matcher text") + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 5 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpPostBody struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index b52adbc6d..68a6d1b01 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -3,6 +3,8 @@ package matchers import ( "testing" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/stretchr/testify/require" ) @@ -71,3 +73,19 @@ func TestHexEncoding(t *testing.T) { require.True(t, isMatched, "Could not match valid Hex condition") require.Equal(t, m.Words, matched) } + +func TestMatcher_MatchDSL(t *testing.T) { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions("contains(body, \"{{VARIABLE}}\")", dsl.HelperFunctions()) + require.Nil(t, err, "couldn't compile expression") + + m := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}} + err = m.CompileMatchers() + require.Nil(t, err, "could not compile matcher") + + values := []string{"PING", "pong"} + + for value := range values { + isMatched := m.MatchDSL(map[string]interface{}{"body": value, "VARIABLE": value}) + require.True(t, isMatched) + } +} From c17aba8646acde90ad2b2d139b4309fd53764150 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 30 Nov 2021 22:41:02 +0530 Subject: [PATCH 184/196] Misc fixes to part definitions --- v2/pkg/protocols/http/http.go | 32 ++++++++++----------- v2/pkg/protocols/offlinehttp/offlinehttp.go | 32 ++++++++++----------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 9acd26b0d..a2a7a8f24 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -171,22 +171,22 @@ type Request struct { // description. Multiple definitions are separated by commas. // Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. var RequestPartDefinitions = map[string]string{ - "template-id": "ID of the template executed", - "template-info": "Info Block of the template executed", - "template-path": "Path of the template executed", - "host": "Host is the input to the template", - "matched": "Matched is the input which was matched upon", - "type": "Type is the type of request made", - "request": "HTTP request made from the client", - "response": "HTTP response recieved from server", - "status_code": "Status Code received from the Server", - "body": "HTTP response body received from server (default)", - "content_length": "HTTP Response content length", - "header,all_headers": "HTTP response headers", - "duration": "HTTP request time duration", - "all": "HTTP response body + headers", - "": "HTTP response cookies in : format", - "": "HTTP response headers in : format", + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "request": "HTTP request made from the client", + "response": "HTTP response recieved from server", + "status_code": "Status Code received from the Server", + "body": "HTTP response body received from server (default)", + "content_length": "HTTP Response content length", + "header,all_headers": "HTTP response headers", + "duration": "HTTP request time duration", + "all": "HTTP response body + headers", + "cookies_from_response": "HTTP response cookies in name:value format", + "headers_from_response": "HTTP response headers in name:value format", } // GetID returns the unique ID of the request if any. diff --git a/v2/pkg/protocols/offlinehttp/offlinehttp.go b/v2/pkg/protocols/offlinehttp/offlinehttp.go index 6f65ac426..328172b65 100644 --- a/v2/pkg/protocols/offlinehttp/offlinehttp.go +++ b/v2/pkg/protocols/offlinehttp/offlinehttp.go @@ -17,22 +17,22 @@ type Request struct { // description. Multiple definitions are separated by commas. // Definitions not having a name (generated on runtime) are prefixed & suffixed by <>. var RequestPartDefinitions = map[string]string{ - "template-id": "ID of the template executed", - "template-info": "Info Block of the template executed", - "template-path": "Path of the template executed", - "host": "Host is the input to the template", - "matched": "Matched is the input which was matched upon", - "type": "Type is the type of request made", - "request": "HTTP request made from the client", - "response": "HTTP response recieved from server", - "status_code": "Status Code received from the Server", - "body": "HTTP response body received from server (default)", - "content_length": "HTTP Response content length", - "header,all_headers": "HTTP response headers", - "duration": "HTTP request time duration", - "all": "HTTP response body + headers", - "": "HTTP response cookies in : format", - "": "HTTP response headers in : format", + "template-id": "ID of the template executed", + "template-info": "Info Block of the template executed", + "template-path": "Path of the template executed", + "host": "Host is the input to the template", + "matched": "Matched is the input which was matched upon", + "type": "Type is the type of request made", + "request": "HTTP request made from the client", + "response": "HTTP response recieved from server", + "status_code": "Status Code received from the Server", + "body": "HTTP response body received from server (default)", + "content_length": "HTTP Response content length", + "header,all_headers": "HTTP response headers", + "duration": "HTTP request time duration", + "all": "HTTP response body + headers", + "cookies_from_response": "HTTP response cookies in name:value format", + "headers_from_response": "HTTP response headers in name:value format", } // GetID returns the unique ID of the request if any. From a9f6f043943535a23220a429bffa4e6a4922b7b4 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 30 Nov 2021 17:17:46 +0000 Subject: [PATCH 185/196] Auto Generate Syntax Docs + JSONSchema [Tue Nov 30 17:17:46 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 459 +---------------------------- v2/pkg/templates/templates_doc.go | 474 +----------------------------- 2 files changed, 8 insertions(+), 925 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 983b6ed1a..dbe55e89f 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -563,38 +563,6 @@ Appears in: -
- -
- - Severity - -
-
- - - - -Enum Values: - - - - undefined - - - info - - - low - - - medium - - - high - - - critical -
- -
- - - ## model.Classification @@ -734,26 +702,8 @@ path: method: GET ``` -Part Definitions: -- template-id - ID of the template executed -- template-info - Info Block of the template executed -- template-path - Path of the template executed -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -- type - Type is the type of request made -- request - HTTP request made from the client -- response - HTTP response recieved from server -- status_code - Status Code received from the Server -- body - HTTP response body received from server (default) -- content_length - HTTP Response content length -- header,all_headers - HTTP response headers -- duration - HTTP request time duration -- all - HTTP response body + headers -- - HTTP response cookies in : format -- - HTTP response headers in : format -
@@ -929,7 +879,7 @@ Valid values:
-method HTTPMethodTypeHolder +method HTTPMethodTypeHolder
@@ -1302,7 +1252,7 @@ Appears in:
-type MatcherTypeHolder +type MatcherTypeHolder
@@ -1621,52 +1571,6 @@ Valid values: -## MatcherTypeHolder -MatcherTypeHolder is used to hold internal type of the matcher - -Appears in: - - -- matchers.Matcher.type - - - - - -
- -
- - MatcherType - -
-
- - - - -Enum Values: - - - - word - - - regex - - - binary - - - status - - - size - - - dsl -
- -
- - - - - ## extractors.Extractor Extractor is used to extract part of response using a regex. @@ -1719,7 +1623,7 @@ name: cookie-extractor
-type ExtractorTypeHolder +type ExtractorTypeHolder
@@ -1961,48 +1865,6 @@ Valid values: -## ExtractorTypeHolder -ExtractorTypeHolder is used to hold internal type of the extractor - -Appears in: - - -- extractors.Extractor.type - - - - - -
- -
- - ExtractorType - -
-
- - - - -Enum Values: - - - - regex - - - kval - - - xpath - - - json -
- -
- - - - - ## generators.AttackTypeHolder AttackTypeHolder is used to hold internal type of the protocol @@ -2019,86 +1881,6 @@ Appears in: -
- -
- - AttackType - -
-
- - - - -Enum Values: - - - - batteringram - - - pitchfork - - - clusterbomb -
- -
- - - - - -## HTTPMethodTypeHolder -HTTPMethodTypeHolder is used to hold internal type of the HTTP Method - -Appears in: - - -- http.Request.method - - - - - -
- -
- - HTTPMethodType - -
-
- - - - -Enum Values: - - - - GET - - - GET - - - POST - - - PUT - - - DELETE - - - CONNECT - - - OPTIONS - - - TRACE - - - PATCH - - - PURGE -
- -
- - - ## dns.Request @@ -2123,24 +1905,8 @@ retries: 2 recursion: true ``` -Part Definitions: -- template-id - ID of the template executed -- template-info - Info Block of the template executed -- template-path - Path of the template executed -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -- request - Request contains the DNS request in text format -- type - Type is the type of request made -- rcode - Rcode field returned for the DNS request -- question - Question contains the DNS question field -- extra - Extra contains the DNS response extra field -- answer - Answer contains the DNS response answer field -- ns - NS contains the DNS response NS field -- raw,body,all - Raw contains the raw DNS response (default) -- trace - Trace contains trace data for DNS request if enabled -
@@ -2235,7 +2001,7 @@ name: '{{FQDN}}'
-type DNSRequestTypeHolder +type DNSRequestTypeHolder
@@ -2367,58 +2133,6 @@ Resolvers to use for the dns requests -## DNSRequestTypeHolder -DNSRequestTypeHolder is used to hold internal type of the DNS type - -Appears in: - - -- dns.Request.type - - - - - -
- -
- - DNSRequestType - -
-
- - - - -Enum Values: - - - - A - - - NS - - - DS - - - CNAME - - - SOA - - - PTR - - - MX - - - TXT - - - AAAA -
- -
- - - - - ## file.Request Request contains a File matching mechanism for local disk operations. @@ -2437,17 +2151,8 @@ extensions: - all ``` -Part Definitions: -- template-id - ID of the template executed -- template-info - Info Block of the template executed -- template-path - Path of the template executed -- matched - Matched is the input which was matched upon -- path - Path is the path of file on local filesystem -- type - Type is the type of request made -- raw,body,all,data - Raw contains the raw file contents -
@@ -2635,19 +2340,8 @@ matchers: - zookeeper.version ``` -Part Definitions: -- template-id - ID of the template executed -- template-info - Info Block of the template executed -- template-path - Path of the template executed -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -- type - Type is the type of request made -- request - Network request made from the client -- body,all,data - Network response recieved from server (default) -- raw - Full Network protocol data -
@@ -2886,7 +2580,7 @@ data: hex_decode('50494e47')
-type NetworkInputTypeHolder +type NetworkInputTypeHolder
@@ -2962,44 +2656,6 @@ name: prefix -## NetworkInputTypeHolder -NetworkInputTypeHolder is used to hold internal type of the Network type - -Appears in: - - -- network.Input.type - - - - - -
- -
- - NetworkInputType - -
-
- - - - -Enum Values: - - - - hex - - - text -
- -
- - - - - ## headless.Request Request contains a Headless protocol request to be made from a template @@ -3010,18 +2666,8 @@ Appears in: -Part Definitions: -- template-id - ID of the template executed -- template-info - Info Block of the template executed -- template-path - Path of the template executed -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -- type - Type is the type of request made -- req - Headless request made from the client -- resp,body,data - Headless response recieved from client (default) -
@@ -3171,7 +2817,7 @@ Description is the optional description of the headless action
-action ActionTypeHolder +action ActionTypeHolder
@@ -3186,84 +2832,6 @@ Action is the type of the action to perform. -## ActionTypeHolder -ActionTypeHolder is used to hold internal type of the action - -Appears in: - - -- engine.Action.action - - - - - -
- -
- - ActionType - -
-
- - - - -Enum Values: - - - - navigate - - - script - - - click - - - rightclick - - - text - - - screenshot - - - time - - - select - - - files - - - waitload - - - getresource - - - extract - - - setmethod - - - addheader - - - setheader - - - deleteheader - - - setbody - - - waitevent - - - keyboard - - - debug - - - sleep - - - waitvisible -
- -
- - - - - ## ssl.Request Request is a request for the SSL protocol @@ -3274,15 +2842,8 @@ Appears in: -Part Definitions: -- type - Type is the type of request made -- response - JSON SSL protocol handshake details -- not_after - Timestamp after which the remote cert expires -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -
@@ -3364,16 +2925,8 @@ Appears in: -Part Definitions: -- type - Type is the type of request made -- success - Success specifies whether websocket connection was successful -- request - Websocket request made to the server -- response - Websocket response recieved from the server -- host - Host is the input to the template -- matched - Matched is the input which was matched upon -
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index bc15b3082..d706d495a 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -16,20 +16,14 @@ var ( MODELClassificationDoc encoder.Doc HTTPRequestDoc encoder.Doc MATCHERSMatcherDoc encoder.Doc - MatcherTypeHolderDoc encoder.Doc EXTRACTORSExtractorDoc encoder.Doc - ExtractorTypeHolderDoc encoder.Doc GENERATORSAttackTypeHolderDoc encoder.Doc - HTTPMethodTypeHolderDoc encoder.Doc DNSRequestDoc encoder.Doc - DNSRequestTypeHolderDoc encoder.Doc FILERequestDoc encoder.Doc NETWORKRequestDoc encoder.Doc NETWORKInputDoc encoder.Doc - NetworkInputTypeHolderDoc encoder.Doc HEADLESSRequestDoc encoder.Doc ENGINEActionDoc encoder.Doc - ActionTypeHolderDoc encoder.Doc SSLRequestDoc encoder.Doc WEBSOCKETRequestDoc encoder.Doc WEBSOCKETInputDoc encoder.Doc @@ -236,20 +230,7 @@ func init() { FieldName: "severity", }, } - SEVERITYHolderDoc.Fields = make([]encoder.Doc, 1) - SEVERITYHolderDoc.Fields[0].Name = "" - SEVERITYHolderDoc.Fields[0].Type = "Severity" - SEVERITYHolderDoc.Fields[0].Note = "" - SEVERITYHolderDoc.Fields[0].Description = "" - SEVERITYHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - SEVERITYHolderDoc.Fields[0].EnumFields = []string{ - "undefined", - "info", - "low", - "medium", - "high", - "critical", - } + SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0) MODELClassificationDoc.Type = "model.Classification" MODELClassificationDoc.Comments[encoder.LineComment] = "" @@ -301,72 +282,6 @@ func init() { FieldName: "requests", }, } - HTTPRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "template-id", - Value: "ID of the template executed", - }, - { - Key: "template-info", - Value: "Info Block of the template executed", - }, - { - Key: "template-path", - Value: "Path of the template executed", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "request", - Value: "HTTP request made from the client", - }, - { - Key: "response", - Value: "HTTP response recieved from server", - }, - { - Key: "status_code", - Value: "Status Code received from the Server", - }, - { - Key: "body", - Value: "HTTP response body received from server (default)", - }, - { - Key: "content_length", - Value: "HTTP Response content length", - }, - { - Key: "header,all_headers", - Value: "HTTP response headers", - }, - { - Key: "duration", - Value: "HTTP request time duration", - }, - { - Key: "all", - Value: "HTTP response body + headers", - }, - { - Key: "", - Value: "HTTP response cookies in : format", - }, - { - Key: "", - Value: "HTTP response headers in : format", - }, - } HTTPRequestDoc.Fields = make([]encoder.Doc, 26) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -665,30 +580,6 @@ func init() { "true", } - MatcherTypeHolderDoc.Type = "MatcherTypeHolder" - MatcherTypeHolderDoc.Comments[encoder.LineComment] = " MatcherTypeHolder is used to hold internal type of the matcher" - MatcherTypeHolderDoc.Description = "MatcherTypeHolder is used to hold internal type of the matcher" - MatcherTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "matchers.Matcher", - FieldName: "type", - }, - } - MatcherTypeHolderDoc.Fields = make([]encoder.Doc, 1) - MatcherTypeHolderDoc.Fields[0].Name = "" - MatcherTypeHolderDoc.Fields[0].Type = "MatcherType" - MatcherTypeHolderDoc.Fields[0].Note = "" - MatcherTypeHolderDoc.Fields[0].Description = "" - MatcherTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - MatcherTypeHolderDoc.Fields[0].EnumFields = []string{ - "word", - "regex", - "binary", - "status", - "size", - "dsl", - } - EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." EXTRACTORSExtractorDoc.Description = "Extractor is used to extract part of response using a regex." @@ -803,28 +694,6 @@ func init() { "true", } - ExtractorTypeHolderDoc.Type = "ExtractorTypeHolder" - ExtractorTypeHolderDoc.Comments[encoder.LineComment] = " ExtractorTypeHolder is used to hold internal type of the extractor" - ExtractorTypeHolderDoc.Description = "ExtractorTypeHolder is used to hold internal type of the extractor" - ExtractorTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "extractors.Extractor", - FieldName: "type", - }, - } - ExtractorTypeHolderDoc.Fields = make([]encoder.Doc, 1) - ExtractorTypeHolderDoc.Fields[0].Name = "" - ExtractorTypeHolderDoc.Fields[0].Type = "ExtractorType" - ExtractorTypeHolderDoc.Fields[0].Note = "" - ExtractorTypeHolderDoc.Fields[0].Description = "" - ExtractorTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - ExtractorTypeHolderDoc.Fields[0].EnumFields = []string{ - "regex", - "kval", - "xpath", - "json", - } - GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" @@ -842,45 +711,7 @@ func init() { FieldName: "attack", }, } - GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 1) - GENERATORSAttackTypeHolderDoc.Fields[0].Name = "" - GENERATORSAttackTypeHolderDoc.Fields[0].Type = "AttackType" - GENERATORSAttackTypeHolderDoc.Fields[0].Note = "" - GENERATORSAttackTypeHolderDoc.Fields[0].Description = "" - GENERATORSAttackTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - GENERATORSAttackTypeHolderDoc.Fields[0].EnumFields = []string{ - "batteringram", - "pitchfork", - "clusterbomb", - } - - HTTPMethodTypeHolderDoc.Type = "HTTPMethodTypeHolder" - HTTPMethodTypeHolderDoc.Comments[encoder.LineComment] = " HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" - HTTPMethodTypeHolderDoc.Description = "HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" - HTTPMethodTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "http.Request", - FieldName: "method", - }, - } - HTTPMethodTypeHolderDoc.Fields = make([]encoder.Doc, 1) - HTTPMethodTypeHolderDoc.Fields[0].Name = "" - HTTPMethodTypeHolderDoc.Fields[0].Type = "HTTPMethodType" - HTTPMethodTypeHolderDoc.Fields[0].Note = "" - HTTPMethodTypeHolderDoc.Fields[0].Description = "" - HTTPMethodTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - HTTPMethodTypeHolderDoc.Fields[0].EnumFields = []string{ - "GET", - "GET", - "POST", - "PUT", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PURGE", - } + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" @@ -893,64 +724,6 @@ func init() { FieldName: "dns", }, } - DNSRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "template-id", - Value: "ID of the template executed", - }, - { - Key: "template-info", - Value: "Info Block of the template executed", - }, - { - Key: "template-path", - Value: "Path of the template executed", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - { - Key: "request", - Value: "Request contains the DNS request in text format", - }, - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "rcode", - Value: "Rcode field returned for the DNS request", - }, - { - Key: "question", - Value: "Question contains the DNS question field", - }, - { - Key: "extra", - Value: "Extra contains the DNS response extra field", - }, - { - Key: "answer", - Value: "Answer contains the DNS response answer field", - }, - { - Key: "ns", - Value: "NS contains the DNS response NS field", - }, - { - Key: "raw,body,all", - Value: "Raw contains the raw DNS response (default)", - }, - { - Key: "trace", - Value: "Trace contains trace data for DNS request if enabled", - }, - } DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1031,33 +804,6 @@ func init() { DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" - DNSRequestTypeHolderDoc.Type = "DNSRequestTypeHolder" - DNSRequestTypeHolderDoc.Comments[encoder.LineComment] = " DNSRequestTypeHolder is used to hold internal type of the DNS type" - DNSRequestTypeHolderDoc.Description = "DNSRequestTypeHolder is used to hold internal type of the DNS type" - DNSRequestTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "dns.Request", - FieldName: "type", - }, - } - DNSRequestTypeHolderDoc.Fields = make([]encoder.Doc, 1) - DNSRequestTypeHolderDoc.Fields[0].Name = "" - DNSRequestTypeHolderDoc.Fields[0].Type = "DNSRequestType" - DNSRequestTypeHolderDoc.Fields[0].Note = "" - DNSRequestTypeHolderDoc.Fields[0].Description = "" - DNSRequestTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - DNSRequestTypeHolderDoc.Fields[0].EnumFields = []string{ - "A", - "NS", - "DS", - "CNAME", - "SOA", - "PTR", - "MX", - "TXT", - "AAAA", - } - FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." FILERequestDoc.Description = "Request contains a File matching mechanism for local disk operations." @@ -1069,36 +815,6 @@ func init() { FieldName: "file", }, } - FILERequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "template-id", - Value: "ID of the template executed", - }, - { - Key: "template-info", - Value: "Info Block of the template executed", - }, - { - Key: "template-path", - Value: "Path of the template executed", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - { - Key: "path", - Value: "Path is the path of file on local filesystem", - }, - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "raw,body,all,data", - Value: "Raw contains the raw file contents", - }, - } FILERequestDoc.Fields = make([]encoder.Doc, 8) FILERequestDoc.Fields[0].Name = "matchers" FILERequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1162,44 +878,6 @@ func init() { FieldName: "network", }, } - NETWORKRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "template-id", - Value: "ID of the template executed", - }, - { - Key: "template-info", - Value: "Info Block of the template executed", - }, - { - Key: "template-path", - Value: "Path of the template executed", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "request", - Value: "Network request made from the client", - }, - { - Key: "body,all,data", - Value: "Network response recieved from server (default)", - }, - { - Key: "raw", - Value: "Full Network protocol data", - }, - } NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" @@ -1305,26 +983,6 @@ func init() { NETWORKInputDoc.Fields[3].AddExample("", "prefix") - NetworkInputTypeHolderDoc.Type = "NetworkInputTypeHolder" - NetworkInputTypeHolderDoc.Comments[encoder.LineComment] = " NetworkInputTypeHolder is used to hold internal type of the Network type" - NetworkInputTypeHolderDoc.Description = "NetworkInputTypeHolder is used to hold internal type of the Network type" - NetworkInputTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "network.Input", - FieldName: "type", - }, - } - NetworkInputTypeHolderDoc.Fields = make([]encoder.Doc, 1) - NetworkInputTypeHolderDoc.Fields[0].Name = "" - NetworkInputTypeHolderDoc.Fields[0].Type = "NetworkInputType" - NetworkInputTypeHolderDoc.Fields[0].Note = "" - NetworkInputTypeHolderDoc.Fields[0].Description = "" - NetworkInputTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - NetworkInputTypeHolderDoc.Fields[0].EnumFields = []string{ - "hex", - "text", - } - HEADLESSRequestDoc.Type = "headless.Request" HEADLESSRequestDoc.Comments[encoder.LineComment] = " Request contains a Headless protocol request to be made from a template" HEADLESSRequestDoc.Description = "Request contains a Headless protocol request to be made from a template" @@ -1334,40 +992,6 @@ func init() { FieldName: "headless", }, } - HEADLESSRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "template-id", - Value: "ID of the template executed", - }, - { - Key: "template-info", - Value: "Info Block of the template executed", - }, - { - Key: "template-path", - Value: "Path of the template executed", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "req", - Value: "Headless request made from the client", - }, - { - Key: "resp,body,data", - Value: "Headless response recieved from client (default)", - }, - } HEADLESSRequestDoc.Fields = make([]encoder.Doc, 5) HEADLESSRequestDoc.Fields[0].Name = "id" HEADLESSRequestDoc.Fields[0].Type = "string" @@ -1430,46 +1054,6 @@ func init() { ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." - ActionTypeHolderDoc.Type = "ActionTypeHolder" - ActionTypeHolderDoc.Comments[encoder.LineComment] = " ActionTypeHolder is used to hold internal type of the action" - ActionTypeHolderDoc.Description = "ActionTypeHolder is used to hold internal type of the action" - ActionTypeHolderDoc.AppearsIn = []encoder.Appearance{ - { - TypeName: "engine.Action", - FieldName: "action", - }, - } - ActionTypeHolderDoc.Fields = make([]encoder.Doc, 1) - ActionTypeHolderDoc.Fields[0].Name = "" - ActionTypeHolderDoc.Fields[0].Type = "ActionType" - ActionTypeHolderDoc.Fields[0].Note = "" - ActionTypeHolderDoc.Fields[0].Description = "" - ActionTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" - ActionTypeHolderDoc.Fields[0].EnumFields = []string{ - "navigate", - "script", - "click", - "rightclick", - "text", - "screenshot", - "time", - "select", - "files", - "waitload", - "getresource", - "extract", - "setmethod", - "addheader", - "setheader", - "deleteheader", - "setbody", - "waitevent", - "keyboard", - "debug", - "sleep", - "waitvisible", - } - SSLRequestDoc.Type = "ssl.Request" SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol" SSLRequestDoc.Description = "Request is a request for the SSL protocol" @@ -1479,28 +1063,6 @@ func init() { FieldName: "ssl", }, } - SSLRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "response", - Value: "JSON SSL protocol handshake details", - }, - { - Key: "not_after", - Value: "Timestamp after which the remote cert expires", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - } SSLRequestDoc.Fields = make([]encoder.Doc, 4) SSLRequestDoc.Fields[0].Name = "matchers" SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1536,32 +1098,6 @@ func init() { FieldName: "websocket", }, } - WEBSOCKETRequestDoc.PartDefinitions = []encoder.KeyValue{ - { - Key: "type", - Value: "Type is the type of request made", - }, - { - Key: "success", - Value: "Success specifies whether websocket connection was successful", - }, - { - Key: "request", - Value: "Websocket request made to the server", - }, - { - Key: "response", - Value: "Websocket response recieved from the server", - }, - { - Key: "host", - Value: "Host is the input to the template", - }, - { - Key: "matched", - Value: "Matched is the input which was matched upon", - }, - } WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) WEBSOCKETRequestDoc.Fields[0].Name = "matchers" WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1713,20 +1249,14 @@ func GetTemplateDoc() *encoder.FileDoc { &MODELClassificationDoc, &HTTPRequestDoc, &MATCHERSMatcherDoc, - &MatcherTypeHolderDoc, &EXTRACTORSExtractorDoc, - &ExtractorTypeHolderDoc, &GENERATORSAttackTypeHolderDoc, - &HTTPMethodTypeHolderDoc, &DNSRequestDoc, - &DNSRequestTypeHolderDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, - &NetworkInputTypeHolderDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, - &ActionTypeHolderDoc, &SSLRequestDoc, &WEBSOCKETRequestDoc, &WEBSOCKETInputDoc, From 1150d832b11ad4e0fc39574a8d4e542269ab1274 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 30 Nov 2021 23:23:14 +0530 Subject: [PATCH 186/196] readme update --- README.md | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 427b11e97..2432d8565 100644 --- a/README.md +++ b/README.md @@ -90,20 +90,24 @@ TARGET: -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) TEMPLATES: - -t, -templates string[] template or template directory paths to include in the scan - -nt, -new-templates run only new templates added in latest nuclei-templates release - -w, -workflows string[] workflow or workflow directory paths to include in the scan - -validate validate the passed templates to nuclei - -tl list all available templates + -t, -templates string[] template or template directory paths to include in the scan + -tu, -template-url string[] URL containing list of templates to run + -nt, -new-templates run only new templates added in latest nuclei-templates release + -w, -workflows string[] workflow or workflow directory paths to include in the scan + -wu, -workflow-url string[] URL containing list of workflows to run + -validate validate the passed templates to nuclei + -tl list all available templates FILTERING: -tags string[] execute a subset of templates that contain the provided tags - -etags, -exclude-tags string[] exclude templates with the provided tags -itags, -include-tags string[] tags from the default deny list that permit executing more intrusive templates - -et, -exclude-templates string[] template or template directory paths to exclude + -etags, -exclude-tags string[] exclude templates with the provided tags -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration - -s, -severity value[] Templates to run based on severity. Possible values - info,low,medium,high,critical - -es, -exclude-severity value[] Templates to exclude based on severity. Possible values - info,low,medium,high,critical + -et, -exclude-templates string[] template or template directory paths to exclude + -s, -severity value[] Templates to run based on severity. Possible values info,low,medium,high,critical + -es, -exclude-severity value[] Templates to exclude based on severity. Possible values info,low,medium,high,critical + -pt, -type value[] protocol types to be executed. Possible values dns,file,http,headless,network,workflow,ssl,websocket + -ept, -exclude-type value[] protocol types to not be executed. Possible values dns,file,http,headless,network,workflow,ssl,websocket -a, -author string[] execute templates that are (co-)created by the specified authors OUTPUT: @@ -115,6 +119,7 @@ OUTPUT: -nm, -no-meta don't display match metadata -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) + -ms, -matcher-status show optional match failure status -me, -markdown-export string directory to export results in markdown format -se, -sarif-export string file to export results in SARIF format @@ -126,10 +131,10 @@ CONFIGURATIONS: -r, -resolvers string file containing resolver list for nuclei -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -ev, -env-vars enable environment variables support to be used in template - -cc, -client-cert client certificate file (PEM-encoded) used for authenticating against scanned hosts - -ck, -client-key client key file (PEM-encoded) used for authenticating against scanned hosts - -ca, -client-ca client certificate authority file (PEM-encoded) used for authenticating against scanned hosts + -ev, -env-vars enable environment variables to be used in template + -cc, -client-cert string client certificate file (PEM-encoded) used for authenticating against scanned hosts + -ck, -client-key string client key file (PEM-encoded) used for authenticating against scanned hosts + -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") @@ -141,10 +146,12 @@ INTERACTSH: -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates RATE-LIMIT: - -rl, -rate-limit int maximum number of requests to send per second (default 150) - -rlm, -rate-limit-minute int maximum number of requests to send per minute - -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) - -c, -concurrency int maximum number of templates to be executed in parallel (default 25) + -rl, -rate-limit int maximum number of requests to send per second (default 150) + -rlm, -rate-limit-minute int maximum number of requests to send per minute + -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) + -c, -concurrency int maximum number of templates to be executed in parallel (default 25) + -hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10) + -hc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10) OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) From 6d5146e540665ea59f0b748c5c9fbde9ddc4bee1 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 1 Dec 2021 10:35:18 -0600 Subject: [PATCH 187/196] validate flag updates --- v2/internal/runner/options.go | 2 +- v2/internal/runner/runner.go | 12 ++++++++---- v2/pkg/catalog/loader/loader.go | 4 ++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index f11d2055c..de50701d3 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -116,7 +116,7 @@ func validateOptions(options *types.Options) error { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose { + if options.Verbose || options.Validate { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 055eca1ab..bb764942c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -69,6 +69,8 @@ func New(options *types.Options) (*Runner, error) { } if options.Validate { parsers.ShouldValidate = true + // Does not update the templates when validate flag is used + options.NoUpdateTemplates = true } if err := runner.updateTemplates(); err != nil { gologger.Warning().Msgf("Could not update templates: %s\n", err) @@ -231,10 +233,12 @@ func (r *Runner) RunEnumeration() error { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - ignoreFile := config.ReadIgnoreFile() - r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) - r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) - + // Exclude ignored file for validation + if !r.options.Validate { + ignoreFile := config.ReadIgnoreFile() + r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) + r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + } var cache *hosterrorscache.Cache if r.options.MaxHostError > 0 { cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 1a61cb25b..a1e3b4748 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -141,6 +141,10 @@ func (store *Store) Load() { // ValidateTemplates takes a list of templates and validates them // erroring out on discovering any faulty templates. func (store *Store) ValidateTemplates(templatesList, workflowsList []string) error { + // consider all the templates by default if no templates passed by user + if len(templatesList) == 0 { + templatesList = store.finalTemplates + } templatePaths := store.config.Catalog.GetTemplatesPath(templatesList) workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList) From c807438ecdba1816292c3820491eff98d2a62b34 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 2 Dec 2021 11:19:37 +0100 Subject: [PATCH 188/196] fixing memory allocation for boolean pointer --- v2/pkg/protocols/dns/dns.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index c7dbd3134..d2890f8a8 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -103,7 +103,8 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.Retries = 3 } if request.Recursion == nil { - *request.Recursion = true + recursion := true + request.Recursion = &recursion } dnsClientOptions := &dnsclientpool.Configuration{ Retries: request.Retries, From 9a0ce60a4f439b6eccc0ea828b5364a302bed888 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 2 Dec 2021 10:59:29 +0000 Subject: [PATCH 189/196] Auto Generate Syntax Docs + JSONSchema [Thu Dec 2 10:59:29 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index dbe55e89f..0d39e7fdc 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -1220,6 +1220,19 @@ SkipVariablesCheck skips the check for unresolved variables in request
+
+ +iterate-all bool + +
+
+ +IterateAll iterates all the values extracted from internal extractors + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index aa84681b2..a2beba2c7 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -780,6 +780,11 @@ "type": "boolean", "title": "skip variable checks", "description": "Skips the check for unresolved variables in request" + }, + "iterate-all": { + "type": "boolean", + "title": "iterate all the values", + "description": "Iterates all the values extracted from internal extractors" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d706d495a..6f21ce82c 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -282,7 +282,7 @@ func init() { FieldName: "requests", }, } - HTTPRequestDoc.Fields = make([]encoder.Doc, 26) + HTTPRequestDoc.Fields = make([]encoder.Doc, 27) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[0].Note = "" @@ -442,6 +442,11 @@ func init() { HTTPRequestDoc.Fields[25].Note = "" HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request" HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request" + HTTPRequestDoc.Fields[26].Name = "iterate-all" + HTTPRequestDoc.Fields[26].Type = "bool" + HTTPRequestDoc.Fields[26].Note = "" + HTTPRequestDoc.Fields[26].Description = "IterateAll iterates all the values extracted from internal extractors" + HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors" MATCHERSMatcherDoc.Type = "matchers.Matcher" MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol." From 46a2a4440dd79591b8b670983714449ffd01baf9 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 2 Dec 2021 12:32:20 +0000 Subject: [PATCH 190/196] Auto Generate Syntax Docs + JSONSchema [Thu Dec 2 12:32:20 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 13 +++++++++++++ nuclei-jsonschema.json | 5 +++++ v2/pkg/templates/templates_doc.go | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 0d39e7fdc..08d0774b2 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -271,6 +271,19 @@ Self Contained marks Requests for the template as self-contained
+
+ +stop-at-first-match bool + +
+
+ +Stop execution once first match is found + +
+ +
+ diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index a2beba2c7..156090895 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -1123,6 +1123,11 @@ "type": "boolean", "title": "mark requests as self-contained", "description": "Mark Requests for the template as self-contained" + }, + "stop-at-first-match": { + "type": "boolean", + "title": "stop at first match", + "description": "Stop at first match for the template" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 6f21ce82c..d101c63b6 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -35,7 +35,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 11) + TemplateDoc.Fields = make([]encoder.Doc, 12) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -103,6 +103,11 @@ func init() { TemplateDoc.Fields[10].Note = "" TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained" TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[11].Name = "stop-at-first-match" + TemplateDoc.Fields[11].Type = "bool" + TemplateDoc.Fields[11].Note = "" + TemplateDoc.Fields[11].Description = "Stop execution once first match is found" + TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" From 5eb17833e43b7362a96cc64c017615fcf1c836e9 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 2 Dec 2021 19:06:54 +0000 Subject: [PATCH 191/196] Auto Generate Syntax Docs + JSONSchema [Thu Dec 2 19:06:54 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 459 ++++++++++++++++++++++++++++- v2/pkg/templates/templates_doc.go | 474 +++++++++++++++++++++++++++++- 2 files changed, 925 insertions(+), 8 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 08d0774b2..aed620891 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -576,6 +576,38 @@ Appears in: +
+ +
+ + Severity + +
+
+ + + + +Enum Values: + + + - undefined + + - info + + - low + + - medium + + - high + + - critical +
+ +
+ + + ## model.Classification @@ -715,8 +747,26 @@ path: method: GET ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - HTTP request made from the client +- response - HTTP response recieved from server +- status_code - Status Code received from the Server +- body - HTTP response body received from server (default) +- content_length - HTTP Response content length +- header,all_headers - HTTP response headers +- duration - HTTP request time duration +- all - HTTP response body + headers +- cookies_from_response - HTTP response cookies in name:value format +- headers_from_response - HTTP response headers in name:value format +
@@ -892,7 +942,7 @@ Valid values:
-method HTTPMethodTypeHolder +method HTTPMethodTypeHolder
@@ -1278,7 +1328,7 @@ Appears in:
-type MatcherTypeHolder +type MatcherTypeHolder
@@ -1597,6 +1647,52 @@ Valid values: +## MatcherTypeHolder +MatcherTypeHolder is used to hold internal type of the matcher + +Appears in: + + +- matchers.Matcher.type + + + + + +
+ +
+ + MatcherType + +
+
+ + + + +Enum Values: + + + - word + + - regex + + - binary + + - status + + - size + + - dsl +
+ +
+ + + + + ## extractors.Extractor Extractor is used to extract part of response using a regex. @@ -1649,7 +1745,7 @@ name: cookie-extractor
-type ExtractorTypeHolder +type ExtractorTypeHolder
@@ -1891,6 +1987,48 @@ Valid values: +## ExtractorTypeHolder +ExtractorTypeHolder is used to hold internal type of the extractor + +Appears in: + + +- extractors.Extractor.type + + + + + +
+ +
+ + ExtractorType + +
+
+ + + + +Enum Values: + + + - regex + + - kval + + - xpath + + - json +
+ +
+ + + + + ## generators.AttackTypeHolder AttackTypeHolder is used to hold internal type of the protocol @@ -1907,6 +2045,86 @@ Appears in: +
+ +
+ + AttackType + +
+
+ + + + +Enum Values: + + + - batteringram + + - pitchfork + + - clusterbomb +
+ +
+ + + + + +## HTTPMethodTypeHolder +HTTPMethodTypeHolder is used to hold internal type of the HTTP Method + +Appears in: + + +- http.Request.method + + + + + +
+ +
+ + HTTPMethodType + +
+
+ + + + +Enum Values: + + + - GET + + - GET + + - POST + + - PUT + + - DELETE + + - CONNECT + + - OPTIONS + + - TRACE + + - PATCH + + - PURGE +
+ +
+ + + ## dns.Request @@ -1931,8 +2149,24 @@ retries: 2 recursion: true ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- request - Request contains the DNS request in text format +- type - Type is the type of request made +- rcode - Rcode field returned for the DNS request +- question - Question contains the DNS question field +- extra - Extra contains the DNS response extra field +- answer - Answer contains the DNS response answer field +- ns - NS contains the DNS response NS field +- raw,body,all - Raw contains the raw DNS response (default) +- trace - Trace contains trace data for DNS request if enabled +
@@ -2027,7 +2261,7 @@ name: '{{FQDN}}'
-type DNSRequestTypeHolder +type DNSRequestTypeHolder
@@ -2159,6 +2393,58 @@ Resolvers to use for the dns requests +## DNSRequestTypeHolder +DNSRequestTypeHolder is used to hold internal type of the DNS type + +Appears in: + + +- dns.Request.type + + + + + +
+ +
+ + DNSRequestType + +
+
+ + + + +Enum Values: + + + - A + + - NS + + - DS + + - CNAME + + - SOA + + - PTR + + - MX + + - TXT + + - AAAA +
+ +
+ + + + + ## file.Request Request contains a File matching mechanism for local disk operations. @@ -2177,8 +2463,17 @@ extensions: - all ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- matched - Matched is the input which was matched upon +- path - Path is the path of file on local filesystem +- type - Type is the type of request made +- raw,body,all,data - Raw contains the raw file contents +
@@ -2366,8 +2661,19 @@ matchers: - zookeeper.version ``` +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- request - Network request made from the client +- body,all,data - Network response recieved from server (default) +- raw - Full Network protocol data +
@@ -2606,7 +2912,7 @@ data: hex_decode('50494e47')
-type NetworkInputTypeHolder +type NetworkInputTypeHolder
@@ -2682,6 +2988,44 @@ name: prefix +## NetworkInputTypeHolder +NetworkInputTypeHolder is used to hold internal type of the Network type + +Appears in: + + +- network.Input.type + + + + + +
+ +
+ + NetworkInputType + +
+
+ + + + +Enum Values: + + + - hex + + - text +
+ +
+ + + + + ## headless.Request Request contains a Headless protocol request to be made from a template @@ -2692,8 +3036,18 @@ Appears in: +Part Definitions: +- template-id - ID of the template executed +- template-info - Info Block of the template executed +- template-path - Path of the template executed +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +- type - Type is the type of request made +- req - Headless request made from the client +- resp,body,data - Headless response recieved from client (default) +
@@ -2843,7 +3197,7 @@ Description is the optional description of the headless action
-action ActionTypeHolder +action ActionTypeHolder
@@ -2858,6 +3212,84 @@ Action is the type of the action to perform. +## ActionTypeHolder +ActionTypeHolder is used to hold internal type of the action + +Appears in: + + +- engine.Action.action + + + + + +
+ +
+ + ActionType + +
+
+ + + + +Enum Values: + + + - navigate + + - script + + - click + + - rightclick + + - text + + - screenshot + + - time + + - select + + - files + + - waitload + + - getresource + + - extract + + - setmethod + + - addheader + + - setheader + + - deleteheader + + - setbody + + - waitevent + + - keyboard + + - debug + + - sleep + + - waitvisible +
+ +
+ + + + + ## ssl.Request Request is a request for the SSL protocol @@ -2868,8 +3300,15 @@ Appears in: +Part Definitions: +- type - Type is the type of request made +- response - JSON SSL protocol handshake details +- not_after - Timestamp after which the remote cert expires +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
@@ -2951,8 +3390,16 @@ Appears in: +Part Definitions: +- type - Type is the type of request made +- success - Success specifies whether websocket connection was successful +- request - Websocket request made to the server +- response - Websocket response recieved from the server +- host - Host is the input to the template +- matched - Matched is the input which was matched upon +
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index d101c63b6..c5c77200b 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -16,14 +16,20 @@ var ( MODELClassificationDoc encoder.Doc HTTPRequestDoc encoder.Doc MATCHERSMatcherDoc encoder.Doc + MatcherTypeHolderDoc encoder.Doc EXTRACTORSExtractorDoc encoder.Doc + ExtractorTypeHolderDoc encoder.Doc GENERATORSAttackTypeHolderDoc encoder.Doc + HTTPMethodTypeHolderDoc encoder.Doc DNSRequestDoc encoder.Doc + DNSRequestTypeHolderDoc encoder.Doc FILERequestDoc encoder.Doc NETWORKRequestDoc encoder.Doc NETWORKInputDoc encoder.Doc + NetworkInputTypeHolderDoc encoder.Doc HEADLESSRequestDoc encoder.Doc ENGINEActionDoc encoder.Doc + ActionTypeHolderDoc encoder.Doc SSLRequestDoc encoder.Doc WEBSOCKETRequestDoc encoder.Doc WEBSOCKETInputDoc encoder.Doc @@ -235,7 +241,20 @@ func init() { FieldName: "severity", }, } - SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0) + SEVERITYHolderDoc.Fields = make([]encoder.Doc, 1) + SEVERITYHolderDoc.Fields[0].Name = "" + SEVERITYHolderDoc.Fields[0].Type = "Severity" + SEVERITYHolderDoc.Fields[0].Note = "" + SEVERITYHolderDoc.Fields[0].Description = "" + SEVERITYHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + SEVERITYHolderDoc.Fields[0].EnumFields = []string{ + "undefined", + "info", + "low", + "medium", + "high", + "critical", + } MODELClassificationDoc.Type = "model.Classification" MODELClassificationDoc.Comments[encoder.LineComment] = "" @@ -287,6 +306,72 @@ func init() { FieldName: "requests", }, } + HTTPRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "HTTP request made from the client", + }, + { + Key: "response", + Value: "HTTP response recieved from server", + }, + { + Key: "status_code", + Value: "Status Code received from the Server", + }, + { + Key: "body", + Value: "HTTP response body received from server (default)", + }, + { + Key: "content_length", + Value: "HTTP Response content length", + }, + { + Key: "header,all_headers", + Value: "HTTP response headers", + }, + { + Key: "duration", + Value: "HTTP request time duration", + }, + { + Key: "all", + Value: "HTTP response body + headers", + }, + { + Key: "cookies_from_response", + Value: "HTTP response cookies in name:value format", + }, + { + Key: "headers_from_response", + Value: "HTTP response headers in name:value format", + }, + } HTTPRequestDoc.Fields = make([]encoder.Doc, 27) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -590,6 +675,30 @@ func init() { "true", } + MatcherTypeHolderDoc.Type = "MatcherTypeHolder" + MatcherTypeHolderDoc.Comments[encoder.LineComment] = " MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.Description = "MatcherTypeHolder is used to hold internal type of the matcher" + MatcherTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "matchers.Matcher", + FieldName: "type", + }, + } + MatcherTypeHolderDoc.Fields = make([]encoder.Doc, 1) + MatcherTypeHolderDoc.Fields[0].Name = "" + MatcherTypeHolderDoc.Fields[0].Type = "MatcherType" + MatcherTypeHolderDoc.Fields[0].Note = "" + MatcherTypeHolderDoc.Fields[0].Description = "" + MatcherTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + MatcherTypeHolderDoc.Fields[0].EnumFields = []string{ + "word", + "regex", + "binary", + "status", + "size", + "dsl", + } + EXTRACTORSExtractorDoc.Type = "extractors.Extractor" EXTRACTORSExtractorDoc.Comments[encoder.LineComment] = " Extractor is used to extract part of response using a regex." EXTRACTORSExtractorDoc.Description = "Extractor is used to extract part of response using a regex." @@ -704,6 +813,28 @@ func init() { "true", } + ExtractorTypeHolderDoc.Type = "ExtractorTypeHolder" + ExtractorTypeHolderDoc.Comments[encoder.LineComment] = " ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.Description = "ExtractorTypeHolder is used to hold internal type of the extractor" + ExtractorTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "extractors.Extractor", + FieldName: "type", + }, + } + ExtractorTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ExtractorTypeHolderDoc.Fields[0].Name = "" + ExtractorTypeHolderDoc.Fields[0].Type = "ExtractorType" + ExtractorTypeHolderDoc.Fields[0].Note = "" + ExtractorTypeHolderDoc.Fields[0].Description = "" + ExtractorTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ExtractorTypeHolderDoc.Fields[0].EnumFields = []string{ + "regex", + "kval", + "xpath", + "json", + } + GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" GENERATORSAttackTypeHolderDoc.Description = "AttackTypeHolder is used to hold internal type of the protocol" @@ -721,7 +852,45 @@ func init() { FieldName: "attack", }, } - GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 0) + GENERATORSAttackTypeHolderDoc.Fields = make([]encoder.Doc, 1) + GENERATORSAttackTypeHolderDoc.Fields[0].Name = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Type = "AttackType" + GENERATORSAttackTypeHolderDoc.Fields[0].Note = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Description = "" + GENERATORSAttackTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + GENERATORSAttackTypeHolderDoc.Fields[0].EnumFields = []string{ + "batteringram", + "pitchfork", + "clusterbomb", + } + + HTTPMethodTypeHolderDoc.Type = "HTTPMethodTypeHolder" + HTTPMethodTypeHolderDoc.Comments[encoder.LineComment] = " HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.Description = "HTTPMethodTypeHolder is used to hold internal type of the HTTP Method" + HTTPMethodTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "method", + }, + } + HTTPMethodTypeHolderDoc.Fields = make([]encoder.Doc, 1) + HTTPMethodTypeHolderDoc.Fields[0].Name = "" + HTTPMethodTypeHolderDoc.Fields[0].Type = "HTTPMethodType" + HTTPMethodTypeHolderDoc.Fields[0].Note = "" + HTTPMethodTypeHolderDoc.Fields[0].Description = "" + HTTPMethodTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + HTTPMethodTypeHolderDoc.Fields[0].EnumFields = []string{ + "GET", + "GET", + "POST", + "PUT", + "DELETE", + "CONNECT", + "OPTIONS", + "TRACE", + "PATCH", + "PURGE", + } DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" @@ -734,6 +903,64 @@ func init() { FieldName: "dns", }, } + DNSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "request", + Value: "Request contains the DNS request in text format", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "rcode", + Value: "Rcode field returned for the DNS request", + }, + { + Key: "question", + Value: "Question contains the DNS question field", + }, + { + Key: "extra", + Value: "Extra contains the DNS response extra field", + }, + { + Key: "answer", + Value: "Answer contains the DNS response answer field", + }, + { + Key: "ns", + Value: "NS contains the DNS response NS field", + }, + { + Key: "raw,body,all", + Value: "Raw contains the raw DNS response (default)", + }, + { + Key: "trace", + Value: "Trace contains trace data for DNS request if enabled", + }, + } DNSRequestDoc.Fields = make([]encoder.Doc, 12) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -814,6 +1041,33 @@ func init() { DNSRequestDoc.Fields[11].Description = "Resolvers to use for the dns requests" DNSRequestDoc.Fields[11].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" + DNSRequestTypeHolderDoc.Type = "DNSRequestTypeHolder" + DNSRequestTypeHolderDoc.Comments[encoder.LineComment] = " DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.Description = "DNSRequestTypeHolder is used to hold internal type of the DNS type" + DNSRequestTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "dns.Request", + FieldName: "type", + }, + } + DNSRequestTypeHolderDoc.Fields = make([]encoder.Doc, 1) + DNSRequestTypeHolderDoc.Fields[0].Name = "" + DNSRequestTypeHolderDoc.Fields[0].Type = "DNSRequestType" + DNSRequestTypeHolderDoc.Fields[0].Note = "" + DNSRequestTypeHolderDoc.Fields[0].Description = "" + DNSRequestTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + DNSRequestTypeHolderDoc.Fields[0].EnumFields = []string{ + "A", + "NS", + "DS", + "CNAME", + "SOA", + "PTR", + "MX", + "TXT", + "AAAA", + } + FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." FILERequestDoc.Description = "Request contains a File matching mechanism for local disk operations." @@ -825,6 +1079,36 @@ func init() { FieldName: "file", }, } + FILERequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "path", + Value: "Path is the path of file on local filesystem", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "raw,body,all,data", + Value: "Raw contains the raw file contents", + }, + } FILERequestDoc.Fields = make([]encoder.Doc, 8) FILERequestDoc.Fields[0].Name = "matchers" FILERequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -888,6 +1172,44 @@ func init() { FieldName: "network", }, } + NETWORKRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "request", + Value: "Network request made from the client", + }, + { + Key: "body,all,data", + Value: "Network response recieved from server (default)", + }, + { + Key: "raw", + Value: "Full Network protocol data", + }, + } NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" @@ -993,6 +1315,26 @@ func init() { NETWORKInputDoc.Fields[3].AddExample("", "prefix") + NetworkInputTypeHolderDoc.Type = "NetworkInputTypeHolder" + NetworkInputTypeHolderDoc.Comments[encoder.LineComment] = " NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.Description = "NetworkInputTypeHolder is used to hold internal type of the Network type" + NetworkInputTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "network.Input", + FieldName: "type", + }, + } + NetworkInputTypeHolderDoc.Fields = make([]encoder.Doc, 1) + NetworkInputTypeHolderDoc.Fields[0].Name = "" + NetworkInputTypeHolderDoc.Fields[0].Type = "NetworkInputType" + NetworkInputTypeHolderDoc.Fields[0].Note = "" + NetworkInputTypeHolderDoc.Fields[0].Description = "" + NetworkInputTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + NetworkInputTypeHolderDoc.Fields[0].EnumFields = []string{ + "hex", + "text", + } + HEADLESSRequestDoc.Type = "headless.Request" HEADLESSRequestDoc.Comments[encoder.LineComment] = " Request contains a Headless protocol request to be made from a template" HEADLESSRequestDoc.Description = "Request contains a Headless protocol request to be made from a template" @@ -1002,6 +1344,40 @@ func init() { FieldName: "headless", }, } + HEADLESSRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "template-id", + Value: "ID of the template executed", + }, + { + Key: "template-info", + Value: "Info Block of the template executed", + }, + { + Key: "template-path", + Value: "Path of the template executed", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "req", + Value: "Headless request made from the client", + }, + { + Key: "resp,body,data", + Value: "Headless response recieved from client (default)", + }, + } HEADLESSRequestDoc.Fields = make([]encoder.Doc, 5) HEADLESSRequestDoc.Fields[0].Name = "id" HEADLESSRequestDoc.Fields[0].Type = "string" @@ -1064,6 +1440,46 @@ func init() { ENGINEActionDoc.Fields[3].Description = "Action is the type of the action to perform." ENGINEActionDoc.Fields[3].Comments[encoder.LineComment] = "Action is the type of the action to perform." + ActionTypeHolderDoc.Type = "ActionTypeHolder" + ActionTypeHolderDoc.Comments[encoder.LineComment] = " ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.Description = "ActionTypeHolder is used to hold internal type of the action" + ActionTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "engine.Action", + FieldName: "action", + }, + } + ActionTypeHolderDoc.Fields = make([]encoder.Doc, 1) + ActionTypeHolderDoc.Fields[0].Name = "" + ActionTypeHolderDoc.Fields[0].Type = "ActionType" + ActionTypeHolderDoc.Fields[0].Note = "" + ActionTypeHolderDoc.Fields[0].Description = "" + ActionTypeHolderDoc.Fields[0].Comments[encoder.LineComment] = "" + ActionTypeHolderDoc.Fields[0].EnumFields = []string{ + "navigate", + "script", + "click", + "rightclick", + "text", + "screenshot", + "time", + "select", + "files", + "waitload", + "getresource", + "extract", + "setmethod", + "addheader", + "setheader", + "deleteheader", + "setbody", + "waitevent", + "keyboard", + "debug", + "sleep", + "waitvisible", + } + SSLRequestDoc.Type = "ssl.Request" SSLRequestDoc.Comments[encoder.LineComment] = " Request is a request for the SSL protocol" SSLRequestDoc.Description = "Request is a request for the SSL protocol" @@ -1073,6 +1489,28 @@ func init() { FieldName: "ssl", }, } + SSLRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "response", + Value: "JSON SSL protocol handshake details", + }, + { + Key: "not_after", + Value: "Timestamp after which the remote cert expires", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } SSLRequestDoc.Fields = make([]encoder.Doc, 4) SSLRequestDoc.Fields[0].Name = "matchers" SSLRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1108,6 +1546,32 @@ func init() { FieldName: "websocket", }, } + WEBSOCKETRequestDoc.PartDefinitions = []encoder.KeyValue{ + { + Key: "type", + Value: "Type is the type of request made", + }, + { + Key: "success", + Value: "Success specifies whether websocket connection was successful", + }, + { + Key: "request", + Value: "Websocket request made to the server", + }, + { + Key: "response", + Value: "Websocket response recieved from the server", + }, + { + Key: "host", + Value: "Host is the input to the template", + }, + { + Key: "matched", + Value: "Matched is the input which was matched upon", + }, + } WEBSOCKETRequestDoc.Fields = make([]encoder.Doc, 8) WEBSOCKETRequestDoc.Fields[0].Name = "matchers" WEBSOCKETRequestDoc.Fields[0].Type = "[]matchers.Matcher" @@ -1259,14 +1723,20 @@ func GetTemplateDoc() *encoder.FileDoc { &MODELClassificationDoc, &HTTPRequestDoc, &MATCHERSMatcherDoc, + &MatcherTypeHolderDoc, &EXTRACTORSExtractorDoc, + &ExtractorTypeHolderDoc, &GENERATORSAttackTypeHolderDoc, + &HTTPMethodTypeHolderDoc, &DNSRequestDoc, + &DNSRequestTypeHolderDoc, &FILERequestDoc, &NETWORKRequestDoc, &NETWORKInputDoc, + &NetworkInputTypeHolderDoc, &HEADLESSRequestDoc, &ENGINEActionDoc, + &ActionTypeHolderDoc, &SSLRequestDoc, &WEBSOCKETRequestDoc, &WEBSOCKETInputDoc, From 0ca39bb6d2b8e8aa275644a6431813f5f079b937 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 Dec 2021 03:46:08 +0000 Subject: [PATCH 192/196] Auto Generate Syntax Docs + JSONSchema [Fri Dec 3 03:46:08 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 6 +++--- v2/pkg/templates/templates_doc.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index aed620891..ab5ea9ae9 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -136,7 +136,7 @@ dns: type: CNAME class: inet retries: 2 - recursion: true + recursion: false ``` @@ -2146,7 +2146,7 @@ name: '{{FQDN}}' type: CNAME class: inet retries: 2 -recursion: true +recursion: false ``` Part Definitions: @@ -2365,7 +2365,7 @@ trace-max-recursion: 100
-recursion bool +recursion dns.bool
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index c5c77200b..e1b739e9f 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -1031,7 +1031,7 @@ func init() { DNSRequestDoc.Fields[9].AddExample("Use a retry of 100 to 150 generally", 100) DNSRequestDoc.Fields[10].Name = "recursion" - DNSRequestDoc.Fields[10].Type = "bool" + DNSRequestDoc.Fields[10].Type = "dns.bool" DNSRequestDoc.Fields[10].Note = "" DNSRequestDoc.Fields[10].Description = "Recursion determines if resolver should recurse all records to get fresh results." DNSRequestDoc.Fields[10].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." From 83364211790eefc5d205d9c894d4770510310848 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 4 Dec 2021 18:40:11 +0100 Subject: [PATCH 193/196] Improving headless test cases (#1313) * Adding headless test cases --- .../headless/headless-basic.yaml | 18 +++++ .../headless/headless-extract-values.yaml | 31 +++++++ .../headless/headless-header-action.yaml | 24 ++++++ v2/cmd/integration-test/headless.go | 81 +++++++++++++++++++ v2/cmd/integration-test/integration-test.go | 1 + 5 files changed, 155 insertions(+) create mode 100644 integration_tests/headless/headless-basic.yaml create mode 100644 integration_tests/headless/headless-extract-values.yaml create mode 100644 integration_tests/headless/headless-header-action.yaml create mode 100644 v2/cmd/integration-test/headless.go diff --git a/integration_tests/headless/headless-basic.yaml b/integration_tests/headless/headless-basic.yaml new file mode 100644 index 000000000..cfc7dcb3c --- /dev/null +++ b/integration_tests/headless/headless-basic.yaml @@ -0,0 +1,18 @@ +id: headless-basic +info: + name: Headless Basic + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}/" + + - action: waitload + matchers: + - type: word + words: + - "" \ No newline at end of file diff --git a/integration_tests/headless/headless-extract-values.yaml b/integration_tests/headless/headless-extract-values.yaml new file mode 100644 index 000000000..e780ac32c --- /dev/null +++ b/integration_tests/headless/headless-extract-values.yaml @@ -0,0 +1,31 @@ + +id: headless-extract-values +info: + name: Headless Extract Value + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: navigate + args: + url: "{{BaseURL}}" + - action: waitload + # From headless/extract-urls.yaml + - action: script + name: extract + args: + code: | + '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n' + + matchers: + - type: word + words: + - "test.html" + + extractors: + - type: kval + part: extract + kval: + - extract \ No newline at end of file diff --git a/integration_tests/headless/headless-header-action.yaml b/integration_tests/headless/headless-header-action.yaml new file mode 100644 index 000000000..ca3c329d8 --- /dev/null +++ b/integration_tests/headless/headless-header-action.yaml @@ -0,0 +1,24 @@ +id: headless-header-action +info: + name: Headless Header Action + author: pdteam + severity: info + tags: headless + +headless: + - steps: + - action: setheader + args: + part: request + key: Test + value: test value + + - action: navigate + args: + url: "{{BaseURL}}/" + + - action: waitload + matchers: + - type: word + words: + - "test value" \ No newline at end of file diff --git a/v2/cmd/integration-test/headless.go b/v2/cmd/integration-test/headless.go new file mode 100644 index 000000000..6039fdb64 --- /dev/null +++ b/v2/cmd/integration-test/headless.go @@ -0,0 +1,81 @@ +package main + +import ( + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var headlessTestcases = map[string]testutils.TestCase{ + "headless/headless-basic.yaml": &headlessBasic{}, + "headless/headless-header-action.yaml": &headlessHeaderActions{}, + "headless/headless-extract-values.yaml": &headlessExtractValues{}, +} + +type headlessBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessBasic) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type headlessHeaderActions struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessHeaderActions) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + testValue := r.Header.Get("test") + if r.Header.Get("test") != "" { + _, _ = w.Write([]byte("" + testValue + "")) + } + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type headlessExtractValues struct{} + +// Execute executes a test case and returns an error if occurred +func (h *headlessExtractValues) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("test")) + }) + ts := httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-headless") + if err != nil { + return err + } + if len(results) != 3 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index d55f74e0f..70bb193ac 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -29,6 +29,7 @@ func main() { "workflow": workflowTestcases, "loader": loaderTestcases, "websocket": websocketTestCases, + "headless": headlessTestcases, } for proto, tests := range protocolTests { if protocol == "" || protocol == proto { From 3c88afac0ca24fd7df80de3af20a0c08f316b7e0 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 5 Dec 2021 15:11:14 +0100 Subject: [PATCH 194/196] Fixing payloads path during validation (#1320) * Fixing payloads path during validation * Added GH Action for public template parsing / validation * tracking payload errors as syntax warnings * improving path parsing + introducing hard failure for runtime errors on validation Co-authored-by: sandeep --- .github/workflows/template-validate.yml | 29 +++++++++++++++++++ v2/go.mod | 8 ++--- v2/go.sum | 23 +++------------ v2/internal/runner/runner.go | 3 +- v2/pkg/catalog/loader/loader.go | 2 ++ v2/pkg/parsers/parser.go | 6 ++-- .../protocols/common/generators/validate.go | 14 ++++++--- 7 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/template-validate.yml diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml new file mode 100644 index 000000000..0f78d4de6 --- /dev/null +++ b/.github/workflows/template-validate.yml @@ -0,0 +1,29 @@ +name: 🛠 Template Validate + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Cache Go + id: cache-go + uses: actions/cache@v2 + with: + path: /home/runner/go + key: ${{ runner.os }}-go + + - name: Installing Nuclei + if: steps.cache-go.outputs.cache-hit != 'true' + run: | + go install github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest + + - name: Template Validation + run: | + nuclei -validate + nuclei -validate -w ./workflows \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index 709577509..490908476 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -65,6 +65,8 @@ require ( moul.io/http2curl v1.0.0 ) +require github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e + require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect @@ -77,7 +79,6 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/dave/dst v0.26.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect @@ -92,7 +93,6 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect @@ -126,14 +126,10 @@ require ( github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/mod v0.4.2 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - golang.org/x/tools v0.1.3 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - mvdan.cc/gofumpt v0.1.1 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 731a08159..bd73d8ced 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -163,7 +163,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -596,6 +595,10 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84 h1:+VqGxv8ywpIHwGGSCOcGn/q5kkuB6F1AZtY42I8VnXc= +github.com/projectdiscovery/folderutil v0.0.0-20210804143510-68474319fd84/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= +github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e h1:ozfSeEc5j1f7NCEZAiAskP/KYfBD/TzPmFTIfh+CEwE= +github.com/projectdiscovery/folderutil v0.0.0-20211203091551-e81604e6940e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a h1:EzwVm8i4zmzqZX55vrDtyfogwHh8AAZ3cWCJe4fEduk= github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= @@ -640,22 +643,7 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d h1:sXbcjsLPDgOrlGXgCKbT6MMyH/hTY3OJhhwsyM2bNlI= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125104740-9b1096de655d/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326 h1:/fGqkG8GlfdvlTCfvAoSA/WgEAJrmCnW5qtdd7QXnwA= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125105154-082a0a3cc326/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784 h1:Be2eD7oXNvCbFufVhvkiO5a0SRVN+Ri2V9pXKL2uJp8= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125110259-585ff5584784/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a h1:xGeZvil8Fe5LpGJbTGZGafnnbedNGtVcvLv3nYtmhXQ= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125112758-99d87632e49a/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b h1:oSBnxdyyDU/WpNKUAXrhydgx5+JtDT7KfJR+hOEaBXk= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135034-67f4c31feb2b/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0 h1:mOptvTJ32yUuqQjjSfiPkPCelTWzqnts92uNOZBXZZo= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125135235-2e6dd74132d0/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e h1:0ZxOM0Q0/ESa24L/vq3fxs9YipxfHR4Y3jM/H2ReJ5E= -github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211125140040-b396ca47606e/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8= github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -907,7 +895,6 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1126,7 +1113,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1267,7 +1253,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= -mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index a5aea7e9e..0d33511b8 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -279,7 +279,7 @@ func (r *Runner) RunEnumeration() error { if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil { return err } - if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 { + if stats.GetValue(parsers.SyntaxErrorStats) == 0 && stats.GetValue(parsers.SyntaxWarningStats) == 0 && stats.GetValue(parsers.RuntimeWarningsStats) == 0 { gologger.Info().Msgf("All templates validated successfully\n") } else { return errors.New("encountered errors while performing template validation") @@ -362,6 +362,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { // Display stats for any loaded templates' syntax warnings or errors stats.Display(parsers.SyntaxWarningStats) stats.Display(parsers.SyntaxErrorStats) + stats.Display(parsers.RuntimeWarningsStats) builder := &strings.Builder{} if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 12ddcfc39..23dfd980a 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) // Config contains the configuration options for the loader @@ -218,6 +219,7 @@ func (store *Store) LoadTemplates(templatesList []string) []*templates.Template if loaded { parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions) if err != nil { + stats.Increment(parsers.RuntimeWarningsStats) gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err) } else if parsed != nil { loadedTemplates = append(loadedTemplates, parsed) diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index e26e7488e..c63922466 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -108,8 +108,9 @@ var ( ) const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" + RuntimeWarningsStats = "runtime-warnings" ) func init() { @@ -118,6 +119,7 @@ func init() { stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") + stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") } // ParseTemplate parses a template and returns a *templates.Template structure diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index 305cae092..b04f8034c 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -4,9 +4,9 @@ import ( "errors" "fmt" "os" - "path/filepath" "strings" + "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -26,10 +26,16 @@ func (g *PayloadGenerator) validate(payloads map[string]interface{}, templatePat } changed := false - pathTokens := strings.Split(templatePath, string(os.PathSeparator)) - for i := range pathTokens { - payloadPath := filepath.Join(filepath.Join(pathTokens[:i]...), payloadType) + templatePathInfo, err := folderutil.NewPathInfo(templatePath) + if err != nil { + return err + } + payloadPathsToProbe, err := templatePathInfo.MeshWith(payloadType) + if err != nil { + return err + } + for _, payloadPath := range payloadPathsToProbe { if fileExists(payloadPath) { payloads[name] = payloadPath changed = true From fb3b4fc510f7406dd7722e5b715b3fa32f0102c7 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 5 Dec 2021 19:45:44 +0530 Subject: [PATCH 195/196] version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 87e6c672d..bb5cfb24f 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.4-dev` +const Version = `2.5.4` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From df55f7a2eb591a7949b5c66a9c54db54aa28f450 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 5 Dec 2021 20:14:16 +0530 Subject: [PATCH 196/196] Disabling no-sandbox in headless engine (#1135) * Disabling no-sandbox in headless engine * limiting disabling sandbox to bare minimum * adding warnings related to linux os and root user requirement Co-authored-by: mzack --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 3 +++ v2/pkg/protocols/headless/engine/engine.go | 13 ++++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 059a96167..bd015fede 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,7 +134,7 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "headless", "Headless", - flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), + flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support (root user on linux will disable sandbox)"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0d33511b8..b71587267 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -76,6 +76,9 @@ func New(options *types.Options) (*Runner, error) { gologger.Warning().Msgf("Could not update templates: %s\n", err) } if options.Headless { + if engine.MustDisableSandbox() { + gologger.Warning().Msgf("The current platform and privileged user will run the browser without sandbox\n") + } browser, err := engine.New(options) if err != nil { return nil, err diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index d73e79044..6c1046b67 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "os" + "runtime" "strings" "github.com/corpix/uarand" @@ -44,12 +45,15 @@ func New(options *types.Options) (*Browser, error) { Set("disable-notifications", "true"). Set("hide-scrollbars", "true"). Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). - Set("no-sandbox", "true"). Set("mute-audio", "true"). Set("incognito", "true"). Delete("use-mock-keychain"). UserDataDir(dataStore) + if MustDisableSandbox() { + chromeLauncher = chromeLauncher.NoSandbox(true) + } + if options.UseInstalledChrome { if chromePath, hasChrome := launcher.LookPath(); hasChrome { chromeLauncher.Bin(chromePath) @@ -105,6 +109,13 @@ func New(options *types.Options) (*Browser, error) { return engine, nil } +// MustDisableSandbox determines if the current os and user needs sandbox mode disabled +func MustDisableSandbox() bool { + // linux with root user needs "--no-sandbox" option + // https://github.com/chromium/chromium/blob/c4d3c31083a2e1481253ff2d24298a1dfe19c754/chrome/test/chromedriver/client/chromedriver.py#L209 + return runtime.GOOS == "linux" && os.Geteuid() == 0 +} + // Close closes the browser engine func (b *Browser) Close() { b.engine.Close()