From 5c79319af547a92960107139dec5836df05e4e64 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 00:04:11 +0530 Subject: [PATCH 01/92] New reworked output package for generic output writing --- v2/pkg/output/doc.go | 2 + v2/pkg/output/file_output_writer.go | 41 +++++++++++++ v2/pkg/output/format_json.go | 14 +++++ v2/pkg/output/format_screen.go | 87 +++++++++++++++++++++++++++ v2/pkg/output/output.go | 92 +++++++++++++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 v2/pkg/output/doc.go create mode 100644 v2/pkg/output/file_output_writer.go create mode 100644 v2/pkg/output/format_json.go create mode 100644 v2/pkg/output/format_screen.go create mode 100644 v2/pkg/output/output.go diff --git a/v2/pkg/output/doc.go b/v2/pkg/output/doc.go new file mode 100644 index 000000000..98f05ad30 --- /dev/null +++ b/v2/pkg/output/doc.go @@ -0,0 +1,2 @@ +// Package output implements output writing interfaces for nuclei. +package output diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go new file mode 100644 index 000000000..6c502e48a --- /dev/null +++ b/v2/pkg/output/file_output_writer.go @@ -0,0 +1,41 @@ +package output + +import ( + "bufio" + "os" +) + +// fileWriter is a concurrent file based output writer. +type fileWriter struct { + file *os.File + writer *bufio.Writer +} + +// NewFileOutputWriter creates a new buffered writer for a file +func newFileOutputWriter(file string) (*fileWriter, error) { + output, err := os.Create(file) + if err != nil { + return nil, err + } + return &fileWriter{file: output, writer: bufio.NewWriter(output)}, nil +} + +// WriteString writes an output to the underlying file +func (w *fileWriter) Write(data []byte) error { + _, err := w.writer.Write(data) + if err != nil { + return err + } + if data[len(data)-1] != '\n' { + _, err = w.writer.WriteRune('\n') + } + return err +} + +// Close closes the underlying writer flushing everything to disk +func (w *fileWriter) Close() error { + w.writer.Flush() + //nolint:errcheck // we don't care whether sync failed or succeeded. + w.file.Sync() + return w.file.Close() +} diff --git a/v2/pkg/output/format_json.go b/v2/pkg/output/format_json.go new file mode 100644 index 000000000..93932c2a0 --- /dev/null +++ b/v2/pkg/output/format_json.go @@ -0,0 +1,14 @@ +package output + +import jsoniter "github.com/json-iterator/go" + +var jsoniterCfg jsoniter.API + +func init() { + jsoniterCfg = jsoniter.Config{SortMapKeys: true}.Froze() +} + +// formatJSON formats the output for json based formatting +func (w *StandardWriter) formatJSON(output Event) ([]byte, error) { + return jsoniterCfg.Marshal(output) +} diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go new file mode 100644 index 000000000..53adaf9aa --- /dev/null +++ b/v2/pkg/output/format_screen.go @@ -0,0 +1,87 @@ +package output + +import ( + "bytes" + "errors" + + "github.com/spf13/cast" +) + +// formatScreen formats the output for showing on screen. +func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { + builder := &bytes.Buffer{} + + if !w.noMetadata { + id, ok := output["id"] + if !ok { + return nil, errors.New("no template id found") + } + builder.WriteRune('[') + builder.WriteString(w.aurora.BrightGreen(id.(string)).String()) + + matcherName, ok := output["matcher_name"] + if ok && matcherName != "" { + builder.WriteString(":") + builder.WriteString(w.aurora.BrightGreen(matcherName).Bold().String()) + } + + outputType, ok := output["type"] + if !ok { + return nil, errors.New("no output type found") + } + builder.WriteString("] [") + builder.WriteString(w.aurora.BrightBlue(outputType.(string)).String()) + builder.WriteString("] ") + + severity, ok := output["severity"] + if !ok { + return nil, errors.New("no output severity found") + } + builder.WriteString("[") + builder.WriteString(w.severityMap[severity.(string)]) + builder.WriteString("] ") + } + matched, ok := output["matched"] + if !ok { + return nil, errors.New("no matched url found") + } + builder.WriteString(matched.(string)) + + // If any extractors, write the results + extractedResults, ok := output["extracted_results"] + if ok { + builder.WriteString(" [") + + extractorResults := cast.ToStringSlice(extractedResults) + for i, item := range extractorResults { + builder.WriteString(w.aurora.BrightCyan(item).String()) + + if i != len(extractorResults)-1 { + builder.WriteRune(',') + } + } + builder.WriteString("]") + } + + // Write meta if any + metaResults, ok := output["meta"] + if ok { + builder.WriteString(" [") + + metaResults := cast.ToStringMap(metaResults) + + var first = true + for name, value := range metaResults { + if first { + builder.WriteRune(',') + } + first = false + + builder.WriteString(w.aurora.BrightYellow(name).String()) + builder.WriteRune('=') + builder.WriteString(w.aurora.BrightYellow(cast.ToString(value)).String()) + } + builder.WriteString("]") + } + return builder.Bytes(), nil +} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go new file mode 100644 index 000000000..04ce704af --- /dev/null +++ b/v2/pkg/output/output.go @@ -0,0 +1,92 @@ +package output + +import ( + "os" + "sync" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" +) + +// Writer is an interface which writes output to somewhere for nuclei events. +type Writer interface { + // Close closes the output writer interface + Close() + // Write writes the event to file and/or screen. + Write(Event) error +} + +// StandardWriter is a writer writing output to file and screen for results. +type StandardWriter struct { + json bool + noMetadata bool + aurora aurora.Aurora + outputFile *fileWriter + mutex *sync.Mutex + severityMap map[string]string +} + +const ( + fgOrange uint8 = 208 + undefined string = "undefined" +) + +// Event is a single output structure from nuclei. +type Event map[string]interface{} + +// NewStandardWriter creates a new output writer based on user configurations +func NewStandardWriter(colors, noMetadata, json bool, file string) (*StandardWriter, error) { + colorizer := aurora.NewAurora(colors) + + var outputFile *fileWriter + if file != "" { + output, err := newFileOutputWriter(file) + if err != nil { + return nil, errors.Wrap(err, "could not create output file") + } + outputFile = output + } + severityMap := map[string]string{ + "info": colorizer.Blue("info").String(), + "low": colorizer.Green("low").String(), + "medium": colorizer.Yellow("medium").String(), + "high": colorizer.Index(fgOrange, "high").String(), + "critical": colorizer.Red("critical").String(), + } + writer := &StandardWriter{ + json: json, + noMetadata: noMetadata, + severityMap: severityMap, + aurora: colorizer, + mutex: &sync.Mutex{}, + outputFile: outputFile, + } + return writer, nil +} + +// Write writes the event to file and/or screen. +func (w *StandardWriter) Write(event Event) error { + var data []byte + var err error + + if w.json { + data, err = w.formatJSON(event) + } else { + data, err = w.formatScreen(event) + } + if err != nil { + return errors.Wrap(err, "could not format output") + } + _, _ = os.Stdout.Write(data) + if w.outputFile != nil { + if writeErr := w.outputFile.Write(data); writeErr != nil { + return errors.Wrap(err, "could not write to output") + } + } + return nil +} + +// Close closes the output writing interface +func (w *StandardWriter) Close() { + w.outputFile.Close() +} From e1bbb9d93d9b3399fe6cd6572b896a488af16d47 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 11:58:33 +0530 Subject: [PATCH 02/92] moved tracefile to pkg/output + misC --- v2/internal/bufwriter/bufwriter.go | 74 ---------------------------- v2/internal/tracelog/tracelog.go | 76 ----------------------------- v2/pkg/output/file_output_writer.go | 4 +- v2/pkg/output/output.go | 62 +++++++++++++++++++++-- 4 files changed, 59 insertions(+), 157 deletions(-) delete mode 100644 v2/internal/bufwriter/bufwriter.go delete mode 100644 v2/internal/tracelog/tracelog.go diff --git a/v2/internal/bufwriter/bufwriter.go b/v2/internal/bufwriter/bufwriter.go deleted file mode 100644 index 32592d482..000000000 --- a/v2/internal/bufwriter/bufwriter.go +++ /dev/null @@ -1,74 +0,0 @@ -package bufwriter - -import ( - "bufio" - "os" - "sync" -) - -// Writer is a mutex protected buffered writer -type Writer struct { - file *os.File - writer *bufio.Writer - mutex *sync.Mutex -} - -// New creates a new mutex protected buffered writer for a file -func New(file string) (*Writer, error) { - output, err := os.Create(file) - if err != nil { - return nil, err - } - return &Writer{file: output, writer: bufio.NewWriter(output), mutex: &sync.Mutex{}}, nil -} - -// Write writes a byte slice to the underlying file -// -// It also writes a newline if the last byte isn't a newline character. -func (w *Writer) Write(data []byte) error { - if len(data) == 0 { - return nil - } - w.mutex.Lock() - defer w.mutex.Unlock() - - _, err := w.writer.Write(data) - if err != nil { - return err - } - if data[len(data)-1] != '\n' { - _, err = w.writer.WriteRune('\n') - } - return err -} - -// WriteString writes a string to the underlying file -// -// It also writes a newline if the last byte isn't a newline character. -func (w *Writer) WriteString(data string) error { - if data == "" { - return nil - } - w.mutex.Lock() - defer w.mutex.Unlock() - - _, err := w.writer.WriteString(data) - if err != nil { - return err - } - if data[len(data)-1] != '\n' { - _, err = w.writer.WriteRune('\n') - } - return err -} - -// Close closes the underlying writer flushing everything to disk -func (w *Writer) Close() error { - w.mutex.Lock() - defer w.mutex.Unlock() - - w.writer.Flush() - //nolint:errcheck // we don't care whether sync failed or succeeded. - w.file.Sync() - return w.file.Close() -} diff --git a/v2/internal/tracelog/tracelog.go b/v2/internal/tracelog/tracelog.go deleted file mode 100644 index ef8215819..000000000 --- a/v2/internal/tracelog/tracelog.go +++ /dev/null @@ -1,76 +0,0 @@ -package tracelog - -import ( - "os" - "sync" - - jsoniter "github.com/json-iterator/go" -) - -// Log is an interface for logging trace log of all the requests -type Log interface { - // Close closes the log interface flushing data - Close() - // Request writes a log the requests trace log - Request(templateID, url, requestType string, err error) -} - -// NoopLogger is a noop logger that simply does nothing -type NoopLogger struct{} - -// Close closes the log interface flushing data -func (n *NoopLogger) Close() {} - -// Request writes a log the requests trace log -func (n *NoopLogger) Request(templateID, url, requestType string, err error) {} - -// FileLogger is a trace logger that writes request logs to a file. -type FileLogger struct { - encoder *jsoniter.Encoder - file *os.File - mutex *sync.Mutex -} - -// NewFileLogger creates a new file logger structure -func NewFileLogger(path string) (*FileLogger, error) { - file, err := os.Create(path) - if err != nil { - return nil, err - } - return &FileLogger{file: file, encoder: jsoniter.NewEncoder(file), mutex: &sync.Mutex{}}, nil -} - -// Close closes the log interface flushing data -func (f *FileLogger) Close() { - f.mutex.Lock() - defer f.mutex.Unlock() - - f.file.Close() -} - -// JSONRequest is a trace log request written to file -type JSONRequest struct { - ID string `json:"id"` - URL string `json:"url"` - Error string `json:"error"` - Type string `json:"type"` -} - -// Request writes a log the requests trace log -func (f *FileLogger) Request(templateID, url, requestType string, err error) { - request := &JSONRequest{ - ID: templateID, - URL: url, - Type: requestType, - } - if err != nil { - request.Error = err.Error() - } else { - request.Error = "none" - } - - f.mutex.Lock() - defer f.mutex.Unlock() - //nolint:errcheck // We don't need to do anything here - f.encoder.Encode(request) -} diff --git a/v2/pkg/output/file_output_writer.go b/v2/pkg/output/file_output_writer.go index 6c502e48a..5bb85ba2a 100644 --- a/v2/pkg/output/file_output_writer.go +++ b/v2/pkg/output/file_output_writer.go @@ -26,9 +26,7 @@ func (w *fileWriter) Write(data []byte) error { if err != nil { return err } - if data[len(data)-1] != '\n' { - _, err = w.writer.WriteRune('\n') - } + _, err = w.writer.WriteRune('\n') return err } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 04ce704af..42edf4bd7 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -4,6 +4,7 @@ import ( "os" "sync" + jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" "github.com/pkg/errors" ) @@ -14,6 +15,8 @@ type Writer interface { Close() // Write writes the event to file and/or screen. Write(Event) error + // Request writes a log the requests trace log + Request(templateID, url, requestType string, err error) } // StandardWriter is a writer writing output to file and screen for results. @@ -22,7 +25,9 @@ type StandardWriter struct { noMetadata bool aurora aurora.Aurora outputFile *fileWriter - mutex *sync.Mutex + outputMutex *sync.Mutex + traceFile *fileWriter + traceMutex *sync.Mutex severityMap map[string]string } @@ -35,7 +40,7 @@ const ( type Event map[string]interface{} // NewStandardWriter creates a new output writer based on user configurations -func NewStandardWriter(colors, noMetadata, json bool, file string) (*StandardWriter, error) { +func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) { colorizer := aurora.NewAurora(colors) var outputFile *fileWriter @@ -46,6 +51,14 @@ func NewStandardWriter(colors, noMetadata, json bool, file string) (*StandardWri } outputFile = output } + var traceOutput *fileWriter + if traceFile != "" { + output, err := newFileOutputWriter(traceFile) + if err != nil { + return nil, errors.Wrap(err, "could not create output file") + } + traceOutput = output + } severityMap := map[string]string{ "info": colorizer.Blue("info").String(), "low": colorizer.Green("low").String(), @@ -58,8 +71,10 @@ func NewStandardWriter(colors, noMetadata, json bool, file string) (*StandardWri noMetadata: noMetadata, severityMap: severityMap, aurora: colorizer, - mutex: &sync.Mutex{}, outputFile: outputFile, + outputMutex: &sync.Mutex{}, + traceFile: traceOutput, + traceMutex: &sync.Mutex{}, } return writer, nil } @@ -86,7 +101,46 @@ func (w *StandardWriter) Write(event Event) error { return nil } +// JSONTraceRequest is a trace log request written to file +type JSONTraceRequest struct { + ID string `json:"id"` + URL string `json:"url"` + Error string `json:"error"` + Type string `json:"type"` +} + +// Request writes a log the requests trace log +func (w *StandardWriter) Request(templateID, url, requestType string, err error) { + if w.traceFile == nil { + return + } + request := &JSONTraceRequest{ + ID: templateID, + URL: url, + Type: requestType, + } + if err != nil { + request.Error = err.Error() + } else { + request.Error = "none" + } + + data, err := jsoniter.Marshal(request) + if err != nil { + return + } + w.traceMutex.Lock() + //nolint:errcheck // We don't need to do anything here + _ = w.traceFile.Write(data) + w.traceMutex.Unlock() +} + // Close closes the output writing interface func (w *StandardWriter) Close() { - w.outputFile.Close() + if w.outputFile != nil { + w.outputFile.Close() + } + if w.traceFile != nil { + w.traceFile.Close() + } } From e6958d7aead465d254f62d24d7f3e15c280bea2e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 12:04:33 +0530 Subject: [PATCH 03/92] Moved colorizer stuff to pkg/output --- v2/pkg/colorizer/colorizer.go | 42 ----------------------------------- v2/pkg/output/output.go | 7 ++++++ 2 files changed, 7 insertions(+), 42 deletions(-) delete mode 100644 v2/pkg/colorizer/colorizer.go diff --git a/v2/pkg/colorizer/colorizer.go b/v2/pkg/colorizer/colorizer.go deleted file mode 100644 index 61937bbf6..000000000 --- a/v2/pkg/colorizer/colorizer.go +++ /dev/null @@ -1,42 +0,0 @@ -package colorizer - -import ( - "strings" - - "github.com/logrusorgru/aurora" -) - -const ( - fgOrange uint8 = 208 - undefined string = "undefined" -) - -// NucleiColorizer contains the severity color mapping -type NucleiColorizer struct { - Colorizer aurora.Aurora - SeverityMap map[string]string -} - -// NewNucleiColorizer initializes the new nuclei colorizer -func NewNucleiColorizer(colorizer aurora.Aurora) *NucleiColorizer { - return &NucleiColorizer{ - Colorizer: colorizer, - SeverityMap: map[string]string{ - "info": colorizer.Blue("info").String(), - "low": colorizer.Green("low").String(), - "medium": colorizer.Yellow("medium").String(), - "high": colorizer.Index(fgOrange, "high").String(), - "critical": colorizer.Red("critical").String(), - }, - } -} - -// GetColorizedSeverity returns the colorized severity string -func (r *NucleiColorizer) GetColorizedSeverity(severity string) string { - sev := r.SeverityMap[strings.ToLower(severity)] - if sev == "" { - return undefined - } - - return sev -} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 42edf4bd7..efaf410d4 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -13,6 +13,8 @@ import ( type Writer interface { // Close closes the output writer interface Close() + // Colorizer returns the colorizer instance for writer + Colorizer() aurora.Aurora // Write writes the event to file and/or screen. Write(Event) error // Request writes a log the requests trace log @@ -135,6 +137,11 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) w.traceMutex.Unlock() } +// Colorizer returns the colorizer instance for writer +func (w *StandardWriter) Colorizer() aurora.Aurora { + return w.aurora +} + // Close closes the output writing interface func (w *StandardWriter) Close() { if w.outputFile != nil { From d631074e35d6cbf978669d8b6824d029812b8455 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 14:31:32 +0530 Subject: [PATCH 04/92] Separating matchers, extractors and requests as protocols and operators --- v2/pkg/{ => operators}/extractors/compile.go | 0 v2/pkg/{ => operators}/extractors/doc.go | 0 v2/pkg/{ => operators}/extractors/extract.go | 0 .../{ => operators}/extractors/extractors.go | 0 v2/pkg/{ => operators}/matchers/compile.go | 0 v2/pkg/{ => operators}/matchers/doc.go | 0 v2/pkg/{ => operators}/matchers/match.go | 5 - v2/pkg/{ => operators}/matchers/match_test.go | 0 v2/pkg/{ => operators}/matchers/matchers.go | 0 v2/pkg/{ => operators}/matchers/util.go | 0 v2/pkg/operators/operators.go | 27 +++++ v2/pkg/protocols/dns/dns.go | 107 ++++++++++++++++++ v2/pkg/protocols/http/http.go | 1 + v2/pkg/protocols/protocols.go | 5 + v2/pkg/types/types.go | 1 + v2/pkg/workflows/var.go | 1 - 16 files changed, 141 insertions(+), 6 deletions(-) rename v2/pkg/{ => operators}/extractors/compile.go (100%) rename v2/pkg/{ => operators}/extractors/doc.go (100%) rename v2/pkg/{ => operators}/extractors/extract.go (100%) rename v2/pkg/{ => operators}/extractors/extractors.go (100%) rename v2/pkg/{ => operators}/matchers/compile.go (100%) rename v2/pkg/{ => operators}/matchers/doc.go (100%) rename v2/pkg/{ => operators}/matchers/match.go (99%) rename v2/pkg/{ => operators}/matchers/match_test.go (100%) rename v2/pkg/{ => operators}/matchers/matchers.go (100%) rename v2/pkg/{ => operators}/matchers/util.go (100%) create mode 100644 v2/pkg/operators/operators.go create mode 100644 v2/pkg/protocols/dns/dns.go create mode 100644 v2/pkg/protocols/http/http.go create mode 100644 v2/pkg/protocols/protocols.go create mode 100644 v2/pkg/types/types.go diff --git a/v2/pkg/extractors/compile.go b/v2/pkg/operators/extractors/compile.go similarity index 100% rename from v2/pkg/extractors/compile.go rename to v2/pkg/operators/extractors/compile.go diff --git a/v2/pkg/extractors/doc.go b/v2/pkg/operators/extractors/doc.go similarity index 100% rename from v2/pkg/extractors/doc.go rename to v2/pkg/operators/extractors/doc.go diff --git a/v2/pkg/extractors/extract.go b/v2/pkg/operators/extractors/extract.go similarity index 100% rename from v2/pkg/extractors/extract.go rename to v2/pkg/operators/extractors/extract.go diff --git a/v2/pkg/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go similarity index 100% rename from v2/pkg/extractors/extractors.go rename to v2/pkg/operators/extractors/extractors.go diff --git a/v2/pkg/matchers/compile.go b/v2/pkg/operators/matchers/compile.go similarity index 100% rename from v2/pkg/matchers/compile.go rename to v2/pkg/operators/matchers/compile.go diff --git a/v2/pkg/matchers/doc.go b/v2/pkg/operators/matchers/doc.go similarity index 100% rename from v2/pkg/matchers/doc.go rename to v2/pkg/operators/matchers/doc.go diff --git a/v2/pkg/matchers/match.go b/v2/pkg/operators/matchers/match.go similarity index 99% rename from v2/pkg/matchers/match.go rename to v2/pkg/operators/matchers/match.go index a2a258409..3584d8d90 100644 --- a/v2/pkg/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -105,7 +105,6 @@ func (m *Matcher) matchSizeCode(length int) bool { // Return on the first match. return true } - return false } @@ -134,7 +133,6 @@ func (m *Matcher) matchWords(corpus string) bool { return true } } - return false } @@ -163,7 +161,6 @@ func (m *Matcher) matchRegex(corpus string) bool { return true } } - return false } @@ -193,7 +190,6 @@ func (m *Matcher) matchBinary(corpus string) bool { return true } } - return false } @@ -230,6 +226,5 @@ func (m *Matcher) matchDSL(mp map[string]interface{}) bool { return true } } - return false } diff --git a/v2/pkg/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go similarity index 100% rename from v2/pkg/matchers/match_test.go rename to v2/pkg/operators/matchers/match_test.go diff --git a/v2/pkg/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go similarity index 100% rename from v2/pkg/matchers/matchers.go rename to v2/pkg/operators/matchers/matchers.go diff --git a/v2/pkg/matchers/util.go b/v2/pkg/operators/matchers/util.go similarity index 100% rename from v2/pkg/matchers/util.go rename to v2/pkg/operators/matchers/util.go diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go new file mode 100644 index 000000000..830830e98 --- /dev/null +++ b/v2/pkg/operators/operators.go @@ -0,0 +1,27 @@ +package operators + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" +) + +// Operators contains the operators that can be applied on protocols +type Operators struct { + // Matchers contains the detection mechanism for the request to identify + // whether the request was successful + Matchers []*matchers.Matcher `yaml:"matchers"` + // Extractors contains the extraction mechanism for the request to identify + // and extract parts of the response. + Extractors []*extractors.Extractor `yaml:"extractors"` + // MatchersCondition is the condition of the matchers + // whether to use AND or OR. Default is OR. + MatchersCondition string `yaml:"matchers-condition"` + + // cached variables that may be used along with request. + matchersCondition matchers.ConditionType +} + +// GetMatchersCondition returns the condition for the matchers +func (r *Operators) GetMatchersCondition() matchers.ConditionType { + return r.matchersCondition +} diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go new file mode 100644 index 000000000..91cda7725 --- /dev/null +++ b/v2/pkg/protocols/dns/dns.go @@ -0,0 +1,107 @@ +package dns + +import ( + "strings" + + "github.com/miekg/dns" +) + +// Request contains a DNS protocol request to be made from a template +type Request struct { + // Recursion specifies whether to recurse all the answers. + Recursion bool `yaml:"recursion"` + // Path contains the path/s for the request + Name string `yaml:"name"` + // Type is the type of DNS request to make + Type string `yaml:"type"` + // Class is the class of the DNS request + Class string `yaml:"class"` + // Retries is the number of retries for the DNS request + Retries int `yaml:"retries"` + // Raw contains a raw request + Raw string `yaml:"raw,omitempty"` + + // cache any variables that may be needed for operation. + class uint16 + questionType uint16 +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile() error { + r.class = classToInt(r.Class) + r.questionType = questionTypeToInt(r.Type) + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int64 { + return 1 +} + +// Make returns the request to be sent for the protocol +func (r *Request) Make(domain string) (*dns.Msg, error) { + domain = dns.Fqdn(domain) + + // Build a request on the specified URL + req := new(dns.Msg) + req.Id = dns.Id() + req.RecursionDesired = r.Recursion + + var q dns.Question + + replacer := newReplacer(map[string]interface{}{"FQDN": domain}) + + q.Name = dns.Fqdn(replacer.Replace(r.Name)) + q.Qclass = classToInt(r.Class) + q.Qtype = questionTypeToInt(r.Type) + req.Question = append(req.Question, q) + return req, nil +} + +// questionTypeToInt converts DNS question type to internal representation +func questionTypeToInt(Type string) uint16 { + Type = strings.TrimSpace(strings.ToUpper(Type)) + question := dns.TypeA + + switch Type { + case "A": + question = dns.TypeA + case "NS": + question = dns.TypeNS + case "CNAME": + question = dns.TypeCNAME + case "SOA": + question = dns.TypeSOA + case "PTR": + question = dns.TypePTR + case "MX": + question = dns.TypeMX + case "TXT": + question = dns.TypeTXT + case "AAAA": + question = dns.TypeAAAA + } + return uint16(question) +} + +// classToInt converts a dns class name to it's internal representation +func classToInt(class string) uint16 { + class = strings.TrimSpace(strings.ToUpper(class)) + result := dns.ClassINET + + switch class { + case "INET": + result = dns.ClassINET + case "CSNET": + result = dns.ClassCSNET + case "CHAOS": + result = dns.ClassCHAOS + case "HESIOD": + result = dns.ClassHESIOD + case "NONE": + result = dns.ClassNONE + case "ANY": + result = dns.ClassANY + } + return uint16(result) +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go new file mode 100644 index 000000000..d02cfda64 --- /dev/null +++ b/v2/pkg/protocols/http/http.go @@ -0,0 +1 @@ +package http diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go new file mode 100644 index 000000000..c3a0d532f --- /dev/null +++ b/v2/pkg/protocols/protocols.go @@ -0,0 +1,5 @@ +package protocols + +// Protocol is an interface implemented by a protocol to be templated. +type Protocol interface { +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go new file mode 100644 index 000000000..ab1254f4c --- /dev/null +++ b/v2/pkg/types/types.go @@ -0,0 +1 @@ +package types diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 02765564b..5d304721e 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -229,6 +229,5 @@ func iterableToMapString(t tengo.Object) map[string]string { } } } - return m } From ed84bb187bd11f2c3878bca8bf62fb83117be762 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 15:51:43 +0530 Subject: [PATCH 05/92] Added per protocol responseToDSL function + misc cleanup with operators --- v2/pkg/operators/matchers/compile.go | 11 ---- v2/pkg/operators/matchers/match.go | 16 ++--- v2/pkg/operators/matchers/matchers.go | 65 ++++++-------------- v2/pkg/operators/matchers/util.go | 85 --------------------------- v2/pkg/protocols/dns/matchers.go | 42 +++++++++++++ v2/pkg/protocols/http/matchers.go | 32 ++++++++++ 6 files changed, 99 insertions(+), 152 deletions(-) delete mode 100644 v2/pkg/operators/matchers/util.go create mode 100644 v2/pkg/protocols/dns/matchers.go create mode 100644 v2/pkg/protocols/http/matchers.go diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 03d65fd47..73d17f9e9 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -47,16 +47,5 @@ func (m *Matcher) CompileMatchers() error { } else { m.condition = ORCondition } - - // Setup the part of the request to match, if any. - if m.Part != "" { - m.part, ok = PartTypes[m.Part] - if !ok { - return fmt.Errorf("unknown matcher part specified: %s", m.Part) - } - } else { - m.part = BodyPart - } - return nil } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 3584d8d90..f39f0d0ef 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -19,27 +19,27 @@ func (m *Matcher) Match(resp *http.Response, body, headers string, duration time return m.isNegative(m.matchSizeCode(len(body))) case WordsMatcher: // Match the parts as required for word check - if m.part == BodyPart { + if m.Part == "body" { return m.isNegative(m.matchWords(body)) - } else if m.part == HeaderPart { + } else if m.Part == "header" { return m.isNegative(m.matchWords(headers)) } else { return m.isNegative(m.matchWords(headers) || m.matchWords(body)) } case RegexMatcher: // Match the parts as required for regex check - if m.part == BodyPart { + if m.Part == "body" { return m.isNegative(m.matchRegex(body)) - } else if m.part == HeaderPart { + } else if m.Part == "header" { return m.isNegative(m.matchRegex(headers)) } else { return m.isNegative(m.matchRegex(headers) || m.matchRegex(body)) } case BinaryMatcher: // Match the parts as required for binary characters check - if m.part == BodyPart { + if m.Part == "body" { return m.isNegative(m.matchBinary(body)) - } else if m.part == HeaderPart { + } else if m.Part == "header" { return m.isNegative(m.matchBinary(headers)) } else { return m.isNegative(m.matchBinary(headers) || m.matchBinary(body)) @@ -55,7 +55,8 @@ func (m *Matcher) Match(resp *http.Response, body, headers string, duration time // MatchDNS matches a dns response against a given matcher func (m *Matcher) MatchDNS(msg *dns.Msg) bool { switch m.matcherType { - // [WIP] add dns status code matcher + case StatusMatcher: + return m.isNegative(m.matchStatusCode(msg.Rcode)) case SizeMatcher: return m.matchSizeCode(msg.Len()) case WordsMatcher: @@ -71,7 +72,6 @@ func (m *Matcher) MatchDNS(msg *dns.Msg) bool { // Match complex query return m.matchDSL(DNSToMap(msg, "")) } - return false } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index be4b4af38..c825e5e16 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -6,12 +6,21 @@ import ( "github.com/Knetic/govaluate" ) -// Matcher is used to identify whether a template was successful. +// Matcher is used to match a part in the output from a protocol. type Matcher struct { // Type is the type of the matcher Type string `yaml:"type"` - // matcherType is the internal type of the matcher - matcherType MatcherType + // Condition is the optional condition between two matcher variables + // + // By default, the condition is assumed to be OR. + Condition string `yaml:"condition,omitempty"` + + // Part is the part of the data to match + Part string `yaml:"part,omitempty"` + + // Negative specifies if the match should be reversed + // It will only match if the condition is not true. + Negative bool `yaml:"negative,omitempty"` // Name is matcher Name Name string `yaml:"name,omitempty"` @@ -23,32 +32,16 @@ type Matcher struct { Words []string `yaml:"words,omitempty"` // Regex are the regex pattern required to be present in the response Regex []string `yaml:"regex,omitempty"` - // regexCompiled is the compiled variant - regexCompiled []*regexp.Regexp // Binary are the binary characters required to be present in the response Binary []string `yaml:"binary,omitempty"` // DSL are the dsl queries DSL []string `yaml:"dsl,omitempty"` - // dslCompiled is the compiled variant - dslCompiled []*govaluate.EvaluableExpression - // Condition is the optional condition between two matcher variables - // - // By default, the condition is assumed to be OR. - Condition string `yaml:"condition,omitempty"` - // condition is the condition of the matcher - condition ConditionType - - // Part is the part of the request to match - // - // By default, matching is performed in request body. - Part string `yaml:"part,omitempty"` - // part is the part of the request to match - part Part - - // Negative specifies if the match should be reversed - // It will only match if the condition is not true. - Negative bool `yaml:"negative,omitempty"` + // cached data for the compiled matcher + condition ConditionType + matcherType MatcherType + regexCompiled []*regexp.Regexp + dslCompiled []*govaluate.EvaluableExpression } // MatcherType is the type of the matcher specified @@ -95,30 +88,6 @@ var ConditionTypes = map[string]ConditionType{ "or": ORCondition, } -// Part is the part of the request to match -type Part int - -const ( - // BodyPart matches body of the response. - BodyPart Part = iota + 1 - // HeaderPart matches headers of the response. - HeaderPart - // AllPart matches both response body and headers of the response. - AllPart -) - -// PartTypes is an table for conversion of part type from string. -var PartTypes = map[string]Part{ - "body": BodyPart, - "header": HeaderPart, - "all": AllPart, -} - -// GetPart returns the part of the matcher -func (m *Matcher) GetPart() Part { - return m.part -} - // isNegative reverts the results of the match if the matcher // is of type negative. func (m *Matcher) isNegative(data bool) bool { diff --git a/v2/pkg/operators/matchers/util.go b/v2/pkg/operators/matchers/util.go deleted file mode 100644 index 221468912..000000000 --- a/v2/pkg/operators/matchers/util.go +++ /dev/null @@ -1,85 +0,0 @@ -package matchers - -import ( - "fmt" - "net/http" - "net/http/httputil" - "strings" - "time" - - "github.com/miekg/dns" -) - -const defaultFormat = "%s" - -// HTTPToMap Converts HTTP to Matcher Map -func HTTPToMap(resp *http.Response, body, headers string, duration time.Duration, format string) (m map[string]interface{}) { - m = make(map[string]interface{}) - - if format == "" { - format = defaultFormat - } - - m[fmt.Sprintf(format, "content_length")] = resp.ContentLength - m[fmt.Sprintf(format, "status_code")] = resp.StatusCode - - for k, v := range resp.Header { - k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) - m[fmt.Sprintf(format, k)] = strings.Join(v, " ") - } - - m[fmt.Sprintf(format, "all_headers")] = headers - m[fmt.Sprintf(format, "body")] = body - - if r, err := httputil.DumpResponse(resp, true); err == nil { - m[fmt.Sprintf(format, "raw")] = string(r) - } - - // Converts duration to seconds (floating point) for DSL syntax - m[fmt.Sprintf(format, "duration")] = duration.Seconds() - - return m -} - -// DNSToMap Converts DNS to Matcher Map -func DNSToMap(msg *dns.Msg, format string) (m map[string]interface{}) { - m = make(map[string]interface{}) - - if format == "" { - format = defaultFormat - } - - m[fmt.Sprintf(format, "rcode")] = msg.Rcode - - var qs string - - for _, question := range msg.Question { - qs += fmt.Sprintln(question.String()) - } - - m[fmt.Sprintf(format, "question")] = qs - - var exs string - for _, extra := range msg.Extra { - exs += fmt.Sprintln(extra.String()) - } - - m[fmt.Sprintf(format, "extra")] = exs - - var ans string - for _, answer := range msg.Answer { - ans += fmt.Sprintln(answer.String()) - } - - m[fmt.Sprintf(format, "answer")] = ans - - var nss string - for _, ns := range msg.Ns { - nss += fmt.Sprintln(ns.String()) - } - - m[fmt.Sprintf(format, "ns")] = nss - m[fmt.Sprintf(format, "raw")] = msg.String() - - return m -} diff --git a/v2/pkg/protocols/dns/matchers.go b/v2/pkg/protocols/dns/matchers.go new file mode 100644 index 000000000..371f3576d --- /dev/null +++ b/v2/pkg/protocols/dns/matchers.go @@ -0,0 +1,42 @@ +package dns + +import ( + "bytes" + + "github.com/miekg/dns" +) + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func responseToDSLMap(msg *dns.Msg) map[string]interface{} { + data := make(map[string]interface{}, 6) + + data["rcode"] = msg.Rcode + + buffer := &bytes.Buffer{} + for _, question := range msg.Question { + buffer.WriteString(question.String()) + } + data["question"] = buffer.String() + buffer.Reset() + + for _, extra := range msg.Extra { + buffer.WriteString(extra.String()) + } + data["extra"] = buffer.String() + buffer.Reset() + + for _, answer := range msg.Answer { + buffer.WriteString(answer.String()) + } + data["answer"] = buffer.String() + buffer.Reset() + + for _, ns := range msg.Ns { + buffer.WriteString(ns.String()) + } + data["ns"] = buffer.String() + buffer.Reset() + + data["raw"] = msg.String() + return data +} diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/matchers.go new file mode 100644 index 000000000..8b2a0a350 --- /dev/null +++ b/v2/pkg/protocols/http/matchers.go @@ -0,0 +1,32 @@ +package http + +import ( + "net/http" + "net/http/httputil" + "strings" + "time" +) + +// responseToDSLMap converts a HTTP response to a map for use in DSL matching +func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { + data := make(map[string]interface{}, len(extra)+6+len(resp.Header)) + for k, v := range extra { + data[k] = v + } + + data["content_length"] = resp.ContentLength + data["status_code"] = resp.StatusCode + + for k, v := range resp.Header { + k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) + data[k] = strings.Join(v, " ") + } + data["all_headers"] = headers + data["body"] = body + + if r, err := httputil.DumpResponse(resp, true); err == nil { + data["raw"] = string(r) + } + data["duration"] = duration.Seconds() + return data +} From 7dcb6388d47d9fd71173ad06b5480d711961443d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 21 Dec 2020 16:46:25 +0530 Subject: [PATCH 06/92] Raw request parser added in own package + test --- v2/pkg/protocols/http/raw/doc.go | 2 + v2/pkg/protocols/http/raw/raw.go | 118 ++++++++++++++++++++++++++ v2/pkg/protocols/http/raw/raw_test.go | 28 ++++++ 3 files changed, 148 insertions(+) create mode 100644 v2/pkg/protocols/http/raw/doc.go create mode 100644 v2/pkg/protocols/http/raw/raw.go create mode 100644 v2/pkg/protocols/http/raw/raw_test.go diff --git a/v2/pkg/protocols/http/raw/doc.go b/v2/pkg/protocols/http/raw/doc.go new file mode 100644 index 000000000..5a65d1a4b --- /dev/null +++ b/v2/pkg/protocols/http/raw/doc.go @@ -0,0 +1,2 @@ +// Package raw provides raw http request parsing abilities for nuclei. +package raw diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go new file mode 100644 index 000000000..894c61f2c --- /dev/null +++ b/v2/pkg/protocols/http/raw/raw.go @@ -0,0 +1,118 @@ +package raw + +import ( + "bufio" + "fmt" + "io/ioutil" + "net/url" + "strings" +) + +// Request defines a HTTP raw request structure +type Request struct { + Method string + Path string + Data string + Headers map[string]string +} + +// Parse parses the raw request as supplied by the user +func Parse(request string, unsafe bool) (*Request, error) { + reader := bufio.NewReader(strings.NewReader(request)) + + rawRequest := Request{ + Headers: make(map[string]string), + } + + s, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("could not read request: %s", err) + } + + parts := strings.Split(s, " ") + + //nolint:gomnd // this is not a magic number + if len(parts) < 3 { + return nil, fmt.Errorf("malformed request supplied") + } + // Set the request Method + rawRequest.Method = parts[0] + + // Accepts all malformed headers + var key, value string + for { + line, readErr := reader.ReadString('\n') + line = strings.TrimSpace(line) + + if readErr != nil || line == "" { + break + } + + //nolint:gomnd // this is not a magic number + p := strings.SplitN(line, ":", 2) + key = p[0] + if len(p) > 1 { + value = p[1] + } + + // in case of unsafe requests multiple headers should be accepted + // therefore use the full line as key + _, found := rawRequest.Headers[key] + if unsafe && found { + rawRequest.Headers[line] = "" + } else { + rawRequest.Headers[key] = value + } + } + + // Handle case with the full http url in path. In that case, + // ignore any host header that we encounter and use the path as request URL + if !unsafe && strings.HasPrefix(parts[1], "http") { + parsed, parseErr := url.Parse(parts[1]) + if parseErr != nil { + return nil, fmt.Errorf("could not parse request URL: %s", parseErr) + } + + rawRequest.Path = parts[1] + rawRequest.Headers["Host"] = parsed.Host + } else { + rawRequest.Path = parts[1] + } + + // Set the request body + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("could not read request body: %s", err) + } + rawRequest.Data = string(b) + return &rawRequest, nil +} + +// URL returns the full URL for a raw request based on provided metadata +func (r *Request) URL(BaseURL string) (string, error) { + parsed, err := url.Parse(BaseURL) + if err != nil { + return "", err + } + + var hostURL string + if r.Headers["Host"] == "" { + hostURL = parsed.Host + } else { + hostURL = r.Headers["Host"] + } + + if r.Path == "" { + r.Path = parsed.Path + } else if strings.HasPrefix(r.Path, "?") { + r.Path = fmt.Sprintf("%s%s", parsed.Path, r.Path) + } + + builder := &strings.Builder{} + builder.WriteString(parsed.Scheme) + builder.WriteString("://") + builder.WriteString(strings.TrimSpace(hostURL)) + builder.WriteString(r.Path) + URL := builder.String() + return URL, nil +} diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go new file mode 100644 index 000000000..0b00c0c43 --- /dev/null +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -0,0 +1,28 @@ +package raw + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseRawRequest(t *testing.T) { + request, err := Parse(`GET /manager/html HTTP/1.1 +Host: {{Hostname}} +Authorization: Basic {{base64('username:password')}} +User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0 +Accept-Language: en-US,en;q=0.9 +Connection: close`, true) + require.Nil(t, err, "could not parse GET request") + require.Equal(t, "GET", request.Method, "Could not parse GET method request correctly") + require.Equal(t, "/manager/html", request.Path, "Could not parse request path correctly") + + request, err = Parse(`POST /login HTTP/1.1 +Host: {{Hostname}} +Connection: close + +username=admin&password=login`, true) + require.Nil(t, err, "could not parse POST request") + require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly") + require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly") +} From 1fa79d6f1f4b1b5198721b3410d34a2cc2ffb13e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 22 Dec 2020 01:02:38 +0530 Subject: [PATCH 07/92] Added reworked generators package --- .../protocols/common/generators/generators.go | 216 ++++++++++++++++++ .../common/generators/generators_test.go | 18 ++ v2/pkg/protocols/common/generators/load.go | 60 +++++ 3 files changed, 294 insertions(+) create mode 100644 v2/pkg/protocols/common/generators/generators.go create mode 100644 v2/pkg/protocols/common/generators/generators_test.go create mode 100644 v2/pkg/protocols/common/generators/load.go diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go new file mode 100644 index 000000000..279a10635 --- /dev/null +++ b/v2/pkg/protocols/common/generators/generators.go @@ -0,0 +1,216 @@ +// Inspired from https://github.com/ffuf/ffuf/blob/master/pkg/input/input.go + +package generators + +// Generator is the generator struct for generating payloads +type Generator struct { + Type Type + payloads map[string][]string +} + +// Type is type of attack +type Type int + +const ( + // Sniper replaces each variables with values at a time. + Sniper Type = iota + 1 + // PitchFork replaces variables with positional value from multiple wordlists + PitchFork + // ClusterBomb replaces variables with all possible combinations of values + ClusterBomb +) + +// StringToType is an table for conversion of attack type from string. +var StringToType = map[string]Type{ + "sniper": Sniper, + "pitchfork": PitchFork, + "clusterbomb": ClusterBomb, +} + +// New creates a new generator structure for payload generation +func New(payloads map[string]interface{}, Type Type) (*Generator, error) { + compiled, err := loadPayloads(payloads) + if err != nil { + return nil, err + } + return &Generator{Type: Type, payloads: compiled}, nil +} + +// Iterator is a single instance of an iterator for a generator structure +type Iterator struct { + Type Type + position int + msbIterator int + payloads []*payloadIterator +} + +// NewIterator creates a new iterator for the payloads generator +func (g *Generator) NewIterator() *Iterator { + var payloads []*payloadIterator + + for name, values := range g.payloads { + payloads = append(payloads, &payloadIterator{name: name, values: values}) + } + return &Iterator{Type: g.Type, payloads: payloads} +} + +// Next returns true if there are more inputs in iterator +func (i *Iterator) Next() bool { + if i.position >= i.Total() { + return false + } + i.position++ + return true +} + +//Total returns the amount of input combinations available +func (i *Iterator) Total() int { + count := 0 + switch i.Type { + case Sniper: + for _, p := range i.payloads { + if p.Total() > count { + count = p.Total() + } + } + case PitchFork: + for _, p := range i.payloads { + if p.Total() > count { + count = p.Total() + } + } + case ClusterBomb: + count = 1 + for _, p := range i.payloads { + count = count * p.Total() + } + } + return count +} + +// Value returns the next value for an iterator +func (i *Iterator) Value() map[string]interface{} { + switch i.Type { + case Sniper: + return i.sniperValue() + case PitchFork: + return i.pitchforkValue() + case ClusterBomb: + return i.clusterbombValue() + default: + return i.sniperValue() + } +} + +// sniperValue returns a list of all payloads for the iterator +func (i *Iterator) sniperValue() map[string]interface{} { + values := make(map[string]interface{}, len(i.payloads)) + + for _, p := range i.payloads { + if !p.Next() { + p.ResetPosition() + } + values[p.Keyword()] = p.Value() + p.IncrementPosition() + } + return values +} + +// pitchforkValue returns a map of keyword:value pairs in same index +func (i *Iterator) pitchforkValue() map[string]interface{} { + values := make(map[string]interface{}, len(i.payloads)) + + for _, p := range i.payloads { + if !p.Next() { + p.ResetPosition() + } + values[p.Keyword()] = p.Value() + p.IncrementPosition() + } + return values +} + +// clusterbombValue returns a combination of all input pairs in key:value format. +func (i *Iterator) clusterbombValue() map[string]interface{} { + values := make(map[string]interface{}, len(i.payloads)) + + // Should we signal the next InputProvider in the slice to increment + signalNext := false + first := true + for index, p := range i.payloads { + if signalNext { + p.IncrementPosition() + signalNext = false + } + if !p.Next() { + // No more inputs in this inputprovider + if index == i.msbIterator { + // Reset all previous wordlists and increment the msb counter + i.msbIterator++ + i.clusterbombIteratorReset() + // Start again + return i.clusterbombValue() + } + p.ResetPosition() + signalNext = true + } + values[p.Keyword()] = p.Value() + if first { + p.IncrementPosition() + first = false + } + } + return values +} + +func (i *Iterator) clusterbombIteratorReset() { + for index, p := range i.payloads { + if index < i.msbIterator { + p.ResetPosition() + } + if index == i.msbIterator { + p.IncrementPosition() + } + } +} + +// payloadIterator is a single instance of an iterator for a single payload list. +type payloadIterator struct { + index int + name string + values []string +} + +// Next returns true if there are more values in payload iterator +func (i *payloadIterator) Next() bool { + if i.index == len(i.values)-1 { + return false + } + return true +} + +// Position returns the position of reader in payload iterator +func (i *payloadIterator) Position() int { + return i.index +} + +func (i *payloadIterator) ResetPosition() { + i.index = 0 +} + +func (i *payloadIterator) IncrementPosition() { + i.index++ +} + +func (i *payloadIterator) Value() string { + value := i.values[i.index] + return value +} + +func (i *payloadIterator) Total() int { + return len(i.values) +} + +func (i *payloadIterator) Keyword() string { + return i.name +} diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go new file mode 100644 index 000000000..38d9b93ce --- /dev/null +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -0,0 +1,18 @@ +package generators + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSniperGenerator(t *testing.T) { + generator, err := New(map[string]interface{}{"username": []string{"admin", "password", "login", "test"}}, Sniper) + require.Nil(t, err, "could not create generator") + + iterator := generator.NewIterator() + for iterator.Next() { + fmt.Printf("value: %v\n", iterator.Value()) + } +} diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go new file mode 100644 index 000000000..0c44b613c --- /dev/null +++ b/v2/pkg/protocols/common/generators/load.go @@ -0,0 +1,60 @@ +package generators + +import ( + "bufio" + "io" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cast" +) + +// loadPayloads loads the input payloads from a map to a data map +func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) { + loadedPayloads := make(map[string][]string) + + for name, payload := range payloads { + switch pt := payload.(type) { + case string: + elements := strings.Split(pt, "\n") + //golint:gomnd // this is not a magic number + if len(elements) >= 2 { + loadedPayloads[name] = elements + } else { + payloads, err := loadPayloadsFromFile(pt) + if err != nil { + return nil, errors.Wrap(err, "could not load payloads") + } + loadedPayloads[name] = payloads + } + case interface{}: + loadedPayloads[name] = cast.ToStringSlice(pt) + } + } + return loadedPayloads, nil +} + +// loadPayloadsFromFile loads a file to a string slice +func loadPayloadsFromFile(filepath string) ([]string, error) { + var lines []string + + file, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + lines = append(lines, text) + } + if err := scanner.Err(); err != nil && err != io.EOF { + return lines, scanner.Err() + } + return lines, nil +} From 3fc7291e16b112b27991df0e49f3caf41f3d869d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 22 Dec 2020 01:21:05 +0530 Subject: [PATCH 08/92] Fixed generator bugs, test cases --- .../protocols/common/generators/generators.go | 2 +- .../common/generators/generators_test.go | 46 +++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 279a10635..c07dbc324 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -183,7 +183,7 @@ type payloadIterator struct { // Next returns true if there are more values in payload iterator func (i *payloadIterator) Next() bool { - if i.index == len(i.values)-1 { + if i.index >= i.Total() { return false } return true diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 38d9b93ce..7289376c4 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -1,18 +1,58 @@ package generators import ( - "fmt" "testing" "github.com/stretchr/testify/require" ) func TestSniperGenerator(t *testing.T) { - generator, err := New(map[string]interface{}{"username": []string{"admin", "password", "login", "test"}}, Sniper) + usernames := []string{"admin", "password", "login", "test"} + + generator, err := New(map[string]interface{}{"username": usernames}, Sniper) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() + count := 0 for iterator.Next() { - fmt.Printf("value: %v\n", iterator.Value()) + count++ + require.Contains(t, usernames, iterator.Value()["username"], "Could not get correct sniper") } + require.Equal(t, len(usernames), count, "could not get correct sniper counts") +} + +func TestPitchforkGenerator(t *testing.T) { + usernames := []string{"admin", "token"} + passwords := []string{"admin", "password"} + + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork) + require.Nil(t, err, "could not create generator") + + iterator := generator.NewIterator() + count := 0 + for iterator.Next() { + count++ + value := iterator.Value() + require.Contains(t, usernames, value["username"], "Could not get correct pitchfork username") + require.Contains(t, passwords, value["password"], "Could not get correct pitchfork password") + } + require.Equal(t, len(passwords), count, "could not get correct pitchfork counts") +} + +func TestClusterbombGenerator(t *testing.T) { + usernames := []string{"admin"} + passwords := []string{"admin", "password", "token"} + + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb) + require.Nil(t, err, "could not create generator") + + iterator := generator.NewIterator() + count := 0 + for iterator.Next() { + count++ + value := iterator.Value() + require.Contains(t, usernames, value["username"], "Could not get correct clusterbomb username") + require.Contains(t, passwords, value["password"], "Could not get correct clusterbomb password") + } + require.Equal(t, 3, count, "could not get correct clusterbomb counts") } From 5cbfa8eababbd4a60882ec0269e6d0c83215ef1a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 22 Dec 2020 03:54:55 +0530 Subject: [PATCH 09/92] Misc modifications, cleaning up things --- v2/internal/runner/options.go | 137 ++++++++---- v2/pkg/generators/attack.go | 20 -- v2/pkg/generators/clusterbomb.go | 53 ----- v2/pkg/generators/pitchfork.go | 38 ---- v2/pkg/generators/sniper.go | 21 -- v2/pkg/generators/util.go | 205 ------------------ .../common/dsl}/dsl.go | 164 ++++++++------ .../common/generators/generators_test.go | 3 +- .../http/race}/syncedreadcloser.go | 31 ++- v2/pkg/protocols/protocols.go | 8 +- v2/pkg/requests/bulk-http-request.go | 107 --------- v2/pkg/requests/generator.go | 1 - 12 files changed, 205 insertions(+), 583 deletions(-) delete mode 100644 v2/pkg/generators/attack.go delete mode 100644 v2/pkg/generators/clusterbomb.go delete mode 100644 v2/pkg/generators/pitchfork.go delete mode 100644 v2/pkg/generators/sniper.go delete mode 100644 v2/pkg/generators/util.go rename v2/pkg/{generators => operators/common/dsl}/dsl.go (57%) rename v2/pkg/{syncedreadcloser => protocols/http/race}/syncedreadcloser.go (62%) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index daf7ca2b4..8f4519043 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,64 +5,107 @@ import ( "flag" "net/url" "os" + "strings" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/requests" ) -// Options contains the configuration options for tuning -// the template requesting process. +// Options contains the configuration options for nuclei scanner. type Options struct { - Vhost bool // Mark the input specified as VHOST input - RandomAgent bool // Generate random User-Agent - Metrics bool // Metrics enables display of metrics via an http endpoint - Sandbox bool // Sandbox mode allows users to run isolated workflows with system commands disabled - Debug bool // Debug mode allows debugging request/responses for the engine - Silent bool // Silent suppresses any extra text and only writes found URLs on screen. - Version bool // Version specifies if we should just show version and exit - Verbose bool // Verbose flag indicates whether to show verbose output or not - NoColor bool // No-Color disables the colored output. - UpdateTemplates bool // UpdateTemplates updates the templates installed at startup - JSON bool // JSON writes json output to files - JSONRequests bool // write requests/responses for matches in JSON output - EnableProgressBar bool // Enable progrss bar - TemplatesVersion bool // Show the templates installed version - TemplateList bool // List available templates - Stdin bool // Stdin specifies whether stdin input was given to the process - StopAtFirstMatch bool // Stop processing template at first full match (this may break chained requests) - NoMeta bool // Don't display metadata for the matches - Project bool // Nuclei uses project folder to avoid sending same HTTP request multiple times - MetricsPort int // MetricsPort is the port to show metrics on - MaxWorkflowDuration int // MaxWorkflowDuration is the maximum time a workflow can run for a URL - BulkSize int // Number of targets analyzed in parallel for each template - TemplateThreads int // Number of templates executed in parallel - Timeout int // Timeout is the seconds to wait for a response from the server. - Retries int // Retries is the number of times to retry the request - RateLimit int // Rate-Limit of requests per specified target - Threads int // Thread controls the number of concurrent requests to make. - BurpCollaboratorBiid string // Burp Collaborator BIID for polling - ProjectPath string // Nuclei uses a user defined project folder - Severity string // Filter templates based on their severity and only run the matching ones. - Target string // Target is a single URL/Domain to scan usng a template - Targets string // Targets specifies the targets to scan using templates. - Output string // Output is the file to write found subdomains to. - ProxyURL string // ProxyURL is the URL for the proxy server - ProxySocksURL string // ProxySocksURL is the URL for the proxy socks server - TemplatesDirectory string // TemplatesDirectory is the directory to use for storing templates - TraceLogFile string // TraceLogFile specifies a file to write with the trace of all requests - Templates multiStringFlag // Signature specifies the template/templates to use - ExcludedTemplates multiStringFlag // Signature specifies the template/templates to exclude - CustomHeaders requests.CustomHeaders // Custom global headers + // Vhost marks the input specified as VHOST input + Vhost bool + // RandomAgent generates random User-Agent + RandomAgent bool + // Metrics enables display of metrics via an http endpoint + Metrics bool + // Sandbox mode allows users to run isolated workflows with system commands disabled + Sandbox bool + // Debug mode allows debugging request/responses for the engine + Debug bool + // Silent suppresses any extra text and only writes found URLs on screen. + Silent bool + // Version specifies if we should just show version and exit + Version bool + // Verbose flag indicates whether to show verbose output or not + Verbose bool + // No-Color disables the colored output. + NoColor bool + // UpdateTemplates updates the templates installed at startup + UpdateTemplates bool + // JSON writes json output to files + JSON bool + // JSONRequests writes requests/responses for matches in JSON output + JSONRequests bool + // EnableProgressBar enables progress bar + EnableProgressBar bool + // TemplatesVersion shows the templates installed version + TemplatesVersion bool + // TemplateList lists available templates + TemplateList bool + // Stdin specifies whether stdin input was given to the process + Stdin bool + // StopAtFirstMatch stops processing template at first full match (this may break chained requests) + StopAtFirstMatch bool + // NoMeta disables display of metadata for the matches + NoMeta bool + // Project is used to avoid sending same HTTP request multiple times + Project bool + // MetricsPort is the port to show metrics on + MetricsPort int + // MaxWorkflowDuration is the maximum time a workflow can run for a URL + MaxWorkflowDuration int + // BulkSize is the of targets analyzed in parallel for each template + BulkSize int + // TemplateThreads is the number of templates executed in parallel + TemplateThreads int + // Timeout is the seconds to wait for a response from the server. + Timeout int + // Retries is the number of times to retry the request + Retries int + // Rate-Limit is the maximum number of requests per specified target + RateLimit int + // Thread controls the number of concurrent requests to make. + Threads int + // BurpCollaboratorBiid is the Burp Collaborator BIID for polling interactions. + BurpCollaboratorBiid string + // ProjectPath allows nuclei to use a user defined project folder + ProjectPath string + // Severity filters templates based on their severity and only run the matching ones. + Severity string + // Target is a single URL/Domain to scan using a template + Target string + // Targets specifies the targets to scan using templates. + Targets string + // Output is the file to write found results to. + Output string + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string + // TemplatesDirectory is the directory to use for storing templates + TemplatesDirectory string + // TraceLogFile specifies a file to write with the trace of all requests + TraceLogFile string + // Templates specifies the template/templates to use + Templates StringSlice + // ExcludedTemplates specifies the template/templates to exclude + ExcludedTemplates StringSlice + // CustomHeaders is the list of custom global headers to send with each request. + CustomHeaders requests.CustomHeaders } -type multiStringFlag []string +// StringSlice is a slice of strings as input +type StringSlice []string -func (m *multiStringFlag) String() string { - return "" +// String returns the stringified version of string slice +func (s *StringSlice) String() string { + return strings.Join(*s, ",") } -func (m *multiStringFlag) Set(value string) error { - *m = append(*m, value) +// Set appends a value to the string slice +func (s *StringSlice) Set(value string) error { + *s = append(*s, value) return nil } diff --git a/v2/pkg/generators/attack.go b/v2/pkg/generators/attack.go deleted file mode 100644 index cd901d5f3..000000000 --- a/v2/pkg/generators/attack.go +++ /dev/null @@ -1,20 +0,0 @@ -package generators - -// Type is type of attack -type Type int - -const ( - // Sniper attack - each variable replaced with values at a time - Sniper Type = iota + 1 - // PitchFork attack - Each variable replaced with positional value in multiple wordlists - PitchFork - // ClusterBomb attack - Generate all possible combinations of values - ClusterBomb -) - -// AttackTypes is an table for conversion of attack type from string. -var AttackTypes = map[string]Type{ - "sniper": Sniper, - "pitchfork": PitchFork, - "clusterbomb": ClusterBomb, -} diff --git a/v2/pkg/generators/clusterbomb.go b/v2/pkg/generators/clusterbomb.go deleted file mode 100644 index a8bb04cd2..000000000 --- a/v2/pkg/generators/clusterbomb.go +++ /dev/null @@ -1,53 +0,0 @@ -package generators - -// ClusterbombGenerator Attack - Generate all possible combinations from an input map with all values listed -// as slices of the same size -func ClusterbombGenerator(payloads map[string][]string) (out chan map[string]interface{}) { - out = make(chan map[string]interface{}) - - // generator - go func() { - defer close(out) - - var order []string - - var parts [][]string - - for name, wordlist := range payloads { - order = append(order, name) - parts = append(parts, wordlist) - } - - var n = 1 - for _, ar := range parts { - n *= len(ar) - } - - var at = make([]int, len(parts)) - loop: - for { - // increment position counters - for i := len(parts) - 1; i >= 0; i-- { - if at[i] > 0 && at[i] >= len(parts[i]) { - if i == 0 || (i == 1 && at[i-1] == len(parts[0])-1) { - break loop - } - at[i] = 0 - at[i-1]++ - } - } - // construct permutation - item := make(map[string]interface{}) - for i, ar := range parts { - var p = at[i] - if p >= 0 && p < len(ar) { - item[order[i]] = ar[p] - } - } - out <- item - at[len(parts)-1]++ - } - }() - - return out -} diff --git a/v2/pkg/generators/pitchfork.go b/v2/pkg/generators/pitchfork.go deleted file mode 100644 index 97aafdfeb..000000000 --- a/v2/pkg/generators/pitchfork.go +++ /dev/null @@ -1,38 +0,0 @@ -package generators - -// PitchforkGenerator Attack - Generate positional combinations from an input map with all values listed -// as slices of the same size -func PitchforkGenerator(payloads map[string][]string) (out chan map[string]interface{}) { - out = make(chan map[string]interface{}) - - size := 0 - - // check if all wordlists have the same size - for _, wordlist := range payloads { - if size == 0 { - size = len(wordlist) - } - - if len(wordlist) != size { - // set size = 0 and exit the cycle - size = 0 - break - } - } - - // generator - go func() { - defer close(out) - - for i := 0; i < size; i++ { - element := make(map[string]interface{}) - for name, wordlist := range payloads { - element[name] = wordlist[i] - } - - out <- element - } - }() - - return out -} diff --git a/v2/pkg/generators/sniper.go b/v2/pkg/generators/sniper.go deleted file mode 100644 index 5d70c9fc1..000000000 --- a/v2/pkg/generators/sniper.go +++ /dev/null @@ -1,21 +0,0 @@ -package generators - -// SniperGenerator Attack - Generate sequential combinations -func SniperGenerator(payloads map[string][]string) (out chan map[string]interface{}) { - out = make(chan map[string]interface{}) - - // generator - go func() { - defer close(out) - - for name, wordlist := range payloads { - for _, value := range wordlist { - element := CopyMapWithDefaultValue(payloads, "") - element[name] = value - out <- element - } - } - }() - - return out -} diff --git a/v2/pkg/generators/util.go b/v2/pkg/generators/util.go deleted file mode 100644 index b7761a546..000000000 --- a/v2/pkg/generators/util.go +++ /dev/null @@ -1,205 +0,0 @@ -package generators - -import ( - "bufio" - "bytes" - "fmt" - "math/rand" - "os" - "strings" -) - -const two = 2 - -// LoadPayloads creating proper data structure -func LoadPayloads(payloads map[string]interface{}) map[string][]string { - loadedPayloads := make(map[string][]string) - // load all wordlists - for name, payload := range payloads { - switch pt := payload.(type) { - case string: - elements := strings.Split(pt, "\n") - if len(elements) >= two { - loadedPayloads[name] = elements - } else { - loadedPayloads[name] = LoadFile(pt) - } - case []interface{}, interface{}: - vv := payload.([]interface{}) - - var v []string - - for _, vvv := range vv { - v = append(v, fmt.Sprintf("%v", vvv)) - } - - loadedPayloads[name] = v - } - } - - return loadedPayloads -} - -// LoadFile into slice of strings -func LoadFile(filepath string) (lines []string) { - for line := range StreamFile(filepath) { - lines = append(lines, line) - } - - return -} - -// StreamFile content to a chan -func StreamFile(filepath string) (content chan string) { - content = make(chan string) - - go func() { - defer close(content) - - file, err := os.Open(filepath) - - if err != nil { - return - } - defer file.Close() - - // yql filter applied - scanner := bufio.NewScanner(file) - for scanner.Scan() { - content <- scanner.Text() - } - - if err := scanner.Err(); err != nil { - return - } - }() - - return -} - -// MergeMaps into a new one -func MergeMaps(m1, m2 map[string]interface{}) (m map[string]interface{}) { - m = make(map[string]interface{}) - - for k, v := range m1 { - m[k] = v - } - - for k, v := range m2 { - m[k] = v - } - - return -} - -// MergeMapsWithStrings into a new string one -func MergeMapsWithStrings(m1, m2 map[string]string) (m map[string]string) { - m = make(map[string]string) - for k, v := range m1 { - m[k] = v - } - - for k, v := range m2 { - m[k] = v - } - - return -} - -func reverseString(s string) string { - runes := []rune(s) - for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { - runes[i], runes[j] = runes[j], runes[i] - } - - return string(runes) -} - -// CopyMap creates a new copy of an existing map -func CopyMap(originalMap map[string]interface{}) map[string]interface{} { - newMap := make(map[string]interface{}) - for key, value := range originalMap { - newMap[key] = value - } - - return newMap -} - -// CopyMapWithDefaultValue creates a new copy of an existing map and set a default value -func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue interface{}) map[string]interface{} { - newMap := make(map[string]interface{}) - for key := range originalMap { - newMap[key] = defaultValue - } - - return newMap -} - -// StringContainsAnyMapItem verifies is a string contains any value of a map -func StringContainsAnyMapItem(m map[string]interface{}, s string) bool { - for key := range m { - if strings.Contains(s, key) { - return true - } - } - - return false -} - -// TrimDelimiters removes trailing brackets -func TrimDelimiters(s string) string { - return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}") -} - -// FileExists checks if a file exists and is not a directory -func FileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - - return !info.IsDir() -} - -// TrimDelimiters removes trailing brackets -func SliceContins(s []string, k string) bool { - for _, a := range s { - if a == k { - return true - } - } - return false -} - -func TrimAll(s, cutset string) string { - for _, c := range cutset { - s = strings.ReplaceAll(s, string(c), "") - } - return s -} - -func RandSeq(base string, n int) string { - b := make([]rune, n) - for i := range b { - b[i] = rune(base[rand.Intn(len(base))]) - } - return string(b) -} - -func insertInto(s string, interval int, sep rune) string { - var buffer bytes.Buffer - before := interval - 1 - last := len(s) - 1 - for i, char := range s { - buffer.WriteRune(char) - if i%interval == before && i != last { - buffer.WriteRune(sep) - } - } - buffer.WriteRune(sep) - return buffer.String() -} - -func toString(v interface{}) string { - return fmt.Sprint(v) -} diff --git a/v2/pkg/generators/dsl.go b/v2/pkg/operators/common/dsl/dsl.go similarity index 57% rename from v2/pkg/generators/dsl.go rename to v2/pkg/operators/common/dsl/dsl.go index 5899ff652..70e69ac46 100644 --- a/v2/pkg/generators/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -1,6 +1,7 @@ -package generators +package dsl import ( + "bytes" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -18,164 +19,158 @@ import ( "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/collaborator" "github.com/spaolacci/murmur3" + "github.com/spf13/cast" ) const ( + numbers = "1234567890" + letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" withCutSetArgsSize = 2 - withMaxRandArgsSize = withCutSetArgsSize withBaseRandArgsSize = 3 + withMaxRandArgsSize = withCutSetArgsSize ) -var letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -var numbers = "1234567890" +// HelperFunctions contains the dsl helper functions +func HelperFunctions() map[string]govaluate.ExpressionFunction { + functions := make(map[string]govaluate.ExpressionFunction) -// HelperFunctions contains the dsl functions -func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { - functions = make(map[string]govaluate.ExpressionFunction) - - // strings functions["len"] = func(args ...interface{}) (interface{}, error) { - length := len(toString(args[0])) - + length := len(cast.ToString(args[0])) return float64(length), nil } functions["toupper"] = func(args ...interface{}) (interface{}, error) { - return strings.ToUpper(toString(args[0])), nil + return strings.ToUpper(cast.ToString(args[0])), nil } functions["tolower"] = func(args ...interface{}) (interface{}, error) { - return strings.ToLower(toString(args[0])), nil + return strings.ToLower(cast.ToString(args[0])), nil } functions["replace"] = func(args ...interface{}) (interface{}, error) { - return strings.ReplaceAll(toString(args[0]), toString(args[1]), toString(args[2])), nil + return strings.ReplaceAll(cast.ToString(args[0]), cast.ToString(args[1]), cast.ToString(args[2])), nil } functions["replace_regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(toString(args[1])) + compiled, err := regexp.Compile(cast.ToString(args[1])) if err != nil { return nil, err } - return compiled.ReplaceAllString(toString(args[0]), toString(args[2])), nil + return compiled.ReplaceAllString(cast.ToString(args[0]), cast.ToString(args[2])), nil } functions["trim"] = func(args ...interface{}) (interface{}, error) { - return strings.Trim(toString(args[0]), toString(args[2])), nil + return strings.Trim(cast.ToString(args[0]), cast.ToString(args[2])), nil } functions["trimleft"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimLeft(toString(args[0]), toString(args[1])), nil + return strings.TrimLeft(cast.ToString(args[0]), cast.ToString(args[1])), nil } functions["trimright"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimRight(toString(args[0]), toString(args[1])), nil + return strings.TrimRight(cast.ToString(args[0]), cast.ToString(args[1])), nil } functions["trimspace"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSpace(toString(args[0])), nil + return strings.TrimSpace(cast.ToString(args[0])), nil } functions["trimprefix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimPrefix(toString(args[0]), toString(args[1])), nil + return strings.TrimPrefix(cast.ToString(args[0]), cast.ToString(args[1])), nil } functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSuffix(toString(args[0]), toString(args[1])), nil + return strings.TrimSuffix(cast.ToString(args[0]), cast.ToString(args[1])), nil } functions["reverse"] = func(args ...interface{}) (interface{}, error) { - return reverseString(toString(args[0])), nil + return reverseString(cast.ToString(args[0])), nil } // encoding functions["base64"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) + sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0]))) return sEnc, nil } // python encodes to base64 with lines of 76 bytes terminated by new line "\n" functions["base64_py"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(toString(args[0]))) - + sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0]))) return insertInto(sEnc, 76, '\n'), nil } functions["base64_decode"] = func(args ...interface{}) (interface{}, error) { - return base64.StdEncoding.DecodeString(toString(args[0])) + return base64.StdEncoding.DecodeString(cast.ToString(args[0])) } functions["url_encode"] = func(args ...interface{}) (interface{}, error) { - return url.PathEscape(toString(args[0])), nil + return url.PathEscape(cast.ToString(args[0])), nil } functions["url_decode"] = func(args ...interface{}) (interface{}, error) { - return url.PathUnescape(toString(args[0])) + return url.PathUnescape(cast.ToString(args[0])) } functions["hex_encode"] = func(args ...interface{}) (interface{}, error) { - return hex.EncodeToString([]byte(toString(args[0]))), nil + return hex.EncodeToString([]byte(cast.ToString(args[0]))), nil } functions["hex_decode"] = func(args ...interface{}) (interface{}, error) { - hx, _ := hex.DecodeString(toString(args[0])) + hx, _ := hex.DecodeString(cast.ToString(args[0])) return string(hx), nil } functions["html_escape"] = func(args ...interface{}) (interface{}, error) { - return html.EscapeString(toString(args[0])), nil + return html.EscapeString(cast.ToString(args[0])), nil } functions["html_unescape"] = func(args ...interface{}) (interface{}, error) { - return html.UnescapeString(toString(args[0])), nil + return html.UnescapeString(cast.ToString(args[0])), nil } // hashing functions["md5"] = func(args ...interface{}) (interface{}, error) { - hash := md5.Sum([]byte(toString(args[0]))) + hash := md5.Sum([]byte(cast.ToString(args[0]))) return hex.EncodeToString(hash[:]), nil } functions["sha256"] = func(args ...interface{}) (interface{}, error) { h := sha256.New() - _, err := h.Write([]byte(toString(args[0]))) + _, err := h.Write([]byte(cast.ToString(args[0]))) if err != nil { return nil, err } - return hex.EncodeToString(h.Sum(nil)), nil } functions["sha1"] = func(args ...interface{}) (interface{}, error) { h := sha1.New() - _, err := h.Write([]byte(toString(args[0]))) + _, err := h.Write([]byte(cast.ToString(args[0]))) if err != nil { return nil, err } - return hex.EncodeToString(h.Sum(nil)), nil } functions["mmh3"] = func(args ...interface{}) (interface{}, error) { - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(toString(args[0])), 0))), nil + return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(cast.ToString(args[0])), 0))), nil } // search functions["contains"] = func(args ...interface{}) (interface{}, error) { - return strings.Contains(toString(args[0]), toString(args[1])), nil + return strings.Contains(cast.ToString(args[0]), cast.ToString(args[1])), nil } functions["regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(toString(args[0])) + compiled, err := regexp.Compile(cast.ToString(args[0])) if err != nil { return nil, err } - - return compiled.MatchString(toString(args[1])), nil + return compiled.MatchString(cast.ToString(args[1])), nil } // random generators @@ -183,14 +178,12 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { chars := letters + numbers bad := "" if len(args) >= 1 { - chars = toString(args[0]) + chars = cast.ToString(args[0]) } if len(args) >= withCutSetArgsSize { - bad = toString(args[1]) + bad = cast.ToString(args[1]) } - - chars = TrimAll(chars, bad) - + chars = trimAll(chars, bad) return chars[rand.Intn(len(chars))], nil } @@ -203,15 +196,13 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = toString(args[1]) + bad = cast.ToString(args[1]) } if len(args) >= withBaseRandArgsSize { - base = toString(args[2]) + base = cast.ToString(args[2]) } - - base = TrimAll(base, bad) - - return RandSeq(base, l), nil + base = trimAll(base, bad) + return randSeq(base, l), nil } functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) { @@ -223,12 +214,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = toString(args[1]) + bad = cast.ToString(args[1]) } - - chars = TrimAll(chars, bad) - - return RandSeq(chars, l), nil + chars = trimAll(chars, bad) + return randSeq(chars, l), nil } functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) { @@ -240,12 +229,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = toString(args[1]) + bad = cast.ToString(args[1]) } - - chars = TrimAll(chars, bad) - - return RandSeq(chars, l), nil + chars = trimAll(chars, bad) + return randSeq(chars, l), nil } functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) { @@ -257,12 +244,10 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = toString(args[1]) + bad = cast.ToString(args[1]) } - - chars = TrimAll(chars, bad) - - return RandSeq(chars, l), nil + chars = trimAll(chars, bad) + return randSeq(chars, l), nil } functions["rand_int"] = func(args ...interface{}) (interface{}, error) { @@ -275,7 +260,6 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { if len(args) >= withMaxRandArgsSize { max = args[1].(int) } - return rand.Intn(max-min) + min, nil } @@ -289,8 +273,44 @@ func HelperFunctions() (functions map[string]govaluate.ExpressionFunction) { // Collaborator functions["collab"] = func(args ...interface{}) (interface{}, error) { // check if collaborator contains a specific pattern - return collaborator.DefaultCollaborator.Has(toString(args[0])), nil + return collaborator.DefaultCollaborator.Has(cast.ToString(args[0])), nil } - return functions } + +func reverseString(s string) string { + runes := []rune(s) + for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { + runes[i], runes[j] = runes[j], runes[i] + } + return string(runes) +} + +func trimAll(s, cutset string) string { + for _, c := range cutset { + s = strings.ReplaceAll(s, string(c), "") + } + return s +} + +func randSeq(base string, n int) string { + b := make([]rune, n) + for i := range b { + b[i] = rune(base[rand.Intn(len(base))]) + } + return string(b) +} + +func insertInto(s string, interval int, sep rune) string { + var buffer bytes.Buffer + before := interval - 1 + last := len(s) - 1 + for i, char := range s { + buffer.WriteRune(char) + if i%interval == before && i != last { + buffer.WriteRune(sep) + } + } + buffer.WriteRune(sep) + return buffer.String() +} diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 7289376c4..e27f4b85f 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -16,7 +16,8 @@ func TestSniperGenerator(t *testing.T) { count := 0 for iterator.Next() { count++ - require.Contains(t, usernames, iterator.Value()["username"], "Could not get correct sniper") + value := iterator.Value() + require.Contains(t, usernames, value["username"], "Could not get correct sniper") } require.Equal(t, len(usernames), count, "could not get correct sniper counts") } diff --git a/v2/pkg/syncedreadcloser/syncedreadcloser.go b/v2/pkg/protocols/http/race/syncedreadcloser.go similarity index 62% rename from v2/pkg/syncedreadcloser/syncedreadcloser.go rename to v2/pkg/protocols/http/race/syncedreadcloser.go index aa96904bb..bdefaca34 100644 --- a/v2/pkg/syncedreadcloser/syncedreadcloser.go +++ b/v2/pkg/protocols/http/race/syncedreadcloser.go @@ -1,4 +1,4 @@ -package syncedreadcloser +package race import ( "fmt" @@ -7,8 +7,9 @@ import ( "time" ) -// compatible with ReadSeeker -type SyncedReadCloser struct { +// syncedReadCloser is compatible with io.ReadSeeker and performs +// gate-based synced writes to enable race condition testing. +type syncedReadCloser struct { data []byte p int64 length int64 @@ -16,9 +17,9 @@ type SyncedReadCloser struct { enableBlocking bool } -func New(r io.ReadCloser) *SyncedReadCloser { +func newSyncedReadCloser(r io.ReadCloser) *syncedReadCloser { var ( - s SyncedReadCloser + s syncedReadCloser err error ) s.data, err = ioutil.ReadAll(r) @@ -29,32 +30,30 @@ func New(r io.ReadCloser) *SyncedReadCloser { s.length = int64(len(s.data)) s.opengate = make(chan struct{}) s.enableBlocking = true - return &s } -func NewOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *SyncedReadCloser { - s := New(r) +func newOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *syncedReadCloser { + s := newSyncedReadCloser(r) s.OpenGateAfter(d) - return s } -func (s *SyncedReadCloser) SetOpenGate(status bool) { +func (s *syncedReadCloser) SetOpenGate(status bool) { s.enableBlocking = status } -func (s *SyncedReadCloser) OpenGate() { +func (s *syncedReadCloser) OpenGate() { s.opengate <- struct{}{} } -func (s *SyncedReadCloser) OpenGateAfter(d time.Duration) { +func (s *syncedReadCloser) OpenGateAfter(d time.Duration) { time.AfterFunc(d, func() { s.opengate <- struct{}{} }) } -func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) { +func (s *syncedReadCloser) Seek(offset int64, whence int) (int64, error) { var err error switch whence { case io.SeekStart: @@ -75,7 +74,7 @@ func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) { return s.p, err } -func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { +func (s *syncedReadCloser) Read(p []byte) (n int, err error) { // If the data fits in the buffer blocks awaiting the sync instruction if s.p+int64(len(p)) >= s.length && s.enableBlocking { <-s.opengate @@ -88,10 +87,10 @@ func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { return n, err } -func (s *SyncedReadCloser) Close() error { +func (s *syncedReadCloser) Close() error { return nil } -func (s *SyncedReadCloser) Len() int { +func (s *syncedReadCloser) Len() int { return int(s.length) } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index c3a0d532f..49576005d 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,5 +1,9 @@ package protocols -// Protocol is an interface implemented by a protocol to be templated. -type Protocol interface { +// Rule is an interface implemented by a protocol rule +type Rule interface { + // Compile compiles the protocol request for further execution. + Compile() error + // Requests returns the total number of requests the rule will perform + Requests() int64 } diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 780e3b298..1297fd9e2 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -1,7 +1,6 @@ package requests import ( - "bufio" "context" "fmt" "io" @@ -357,112 +356,6 @@ func (c *CustomHeaders) Set(value string) error { return nil } -// RawRequest defines a basic HTTP raw request -type RawRequest struct { - FullURL string - Method string - Path string - Data string - Headers map[string]string -} - -// parseRawRequest parses the raw request as supplied by the user -func (r *BulkHTTPRequest) parseRawRequest(request, baseURL string) (*RawRequest, error) { - reader := bufio.NewReader(strings.NewReader(request)) - - rawRequest := RawRequest{ - Headers: make(map[string]string), - } - - s, err := reader.ReadString('\n') - if err != nil { - return nil, fmt.Errorf("could not read request: %s", err) - } - - parts := strings.Split(s, " ") - - if len(parts) < three { - return nil, fmt.Errorf("malformed request supplied") - } - // Set the request Method - rawRequest.Method = parts[0] - - // Accepts all malformed headers - var key, value string - for { - line, readErr := reader.ReadString('\n') - line = strings.TrimSpace(line) - - if readErr != nil || line == "" { - break - } - - p := strings.SplitN(line, ":", two) - key = p[0] - if len(p) > 1 { - value = p[1] - } - - // in case of unsafe requests multiple headers should be accepted - // therefore use the full line as key - _, found := rawRequest.Headers[key] - if r.Unsafe && found { - rawRequest.Headers[line] = "" - } else { - rawRequest.Headers[key] = value - } - } - - // Handle case with the full http url in path. In that case, - // ignore any host header that we encounter and use the path as request URL - if !r.Unsafe && strings.HasPrefix(parts[1], "http") { - parsed, parseErr := url.Parse(parts[1]) - if parseErr != nil { - return nil, fmt.Errorf("could not parse request URL: %s", parseErr) - } - - rawRequest.Path = parts[1] - rawRequest.Headers["Host"] = parsed.Host - } else { - rawRequest.Path = parts[1] - } - - // If raw request doesn't have a Host header and/ path, - // this will be generated from the parsed baseURL - parsedURL, err := url.Parse(baseURL) - if err != nil { - return nil, fmt.Errorf("could not parse request URL: %s", err) - } - - var hostURL string - if rawRequest.Headers["Host"] == "" { - hostURL = parsedURL.Host - } else { - hostURL = rawRequest.Headers["Host"] - } - - if rawRequest.Path == "" { - rawRequest.Path = parsedURL.Path - } else if strings.HasPrefix(rawRequest.Path, "?") { - // requests generated from http.ReadRequest have incorrect RequestURI, so they - // cannot be used to perform another request directly, we need to generate a new one - // with the new target url - rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) - } - - rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) - - // Set the request body - b, err := ioutil.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("could not read request body: %s", err) - } - - rawRequest.Data = string(b) - - return &rawRequest, nil -} - // Next returns the next generator by URL func (r *BulkHTTPRequest) Next(reqURL string) bool { return r.gsfm.Next(reqURL) diff --git a/v2/pkg/requests/generator.go b/v2/pkg/requests/generator.go index 2f60f0f3c..50e80e4a3 100644 --- a/v2/pkg/requests/generator.go +++ b/v2/pkg/requests/generator.go @@ -246,7 +246,6 @@ func (gfsm *GeneratorFSM) Total() int { estimatedRequestsWithPayload += prod } } - return len(gfsm.Paths) + len(gfsm.Raws) + estimatedRequestsWithPayload } From de5f7e6ee6183da6a80695a38fb4908e720c152d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 22 Dec 2020 04:11:07 +0530 Subject: [PATCH 10/92] Moved collaborator to internal --- v2/internal/collaborator/collaborator.go | 72 ++++++++++++++++++++++++ v2/pkg/collaborator/collaborator.go | 64 --------------------- v2/pkg/collaborator/util.go | 9 --- v2/pkg/protocols/http/http.go | 64 +++++++++++++++++++++ v2/pkg/requests/bulk-http-request.go | 59 ------------------- 5 files changed, 136 insertions(+), 132 deletions(-) create mode 100644 v2/internal/collaborator/collaborator.go delete mode 100644 v2/pkg/collaborator/collaborator.go delete mode 100644 v2/pkg/collaborator/util.go diff --git a/v2/internal/collaborator/collaborator.go b/v2/internal/collaborator/collaborator.go new file mode 100644 index 000000000..076e18a8a --- /dev/null +++ b/v2/internal/collaborator/collaborator.go @@ -0,0 +1,72 @@ +package collaborator + +import ( + "strings" + "sync" + "time" + + "github.com/projectdiscovery/collaborator" +) + +var ( + // PollSeconds is the seconds to poll at. + PollSeconds = 5 + // DefaultMaxBufferLimit is the default request buffer limit + DefaultMaxBufferLimit = 150 + // DefaultPollInterval is the default poll interval for burp collabortor polling. + DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds) + // DefaultCollaborator is the default burp collaborator instance + DefaultCollaborator = &Collaborator{Collab: collaborator.NewBurpCollaborator()} +) + +// Collaborator is a client for recording burp collaborator interactions +type Collaborator struct { + sync.RWMutex + options *Options // unused + Collab *collaborator.BurpCollaborator +} + +// Options contains configuration options for collaborator client +type Options struct { + BIID string + PollInterval time.Duration + MaxBufferLimit int +} + +// New creates a new collaborator client +func New(options *Options) *Collaborator { + collab := collaborator.NewBurpCollaborator() + collab.AddBIID(options.BIID) + collab.MaxBufferLimit = options.MaxBufferLimit + return &Collaborator{Collab: collab, options: options} +} + +// Poll initiates collaborator polling if any BIIDs were provided +func (b *Collaborator) Poll() { + // if no valid biids were provided just return + if len(b.Collab.BIIDs) > 0 { + go b.Collab.PollEach(DefaultPollInterval) + } +} + +// Has checks if a collabrator hit was found for a URL +func (b *Collaborator) Has(s string) bool { + for _, r := range b.Collab.RespBuffer { + for i := 0; i < len(r.Responses); i++ { + // search in dns - http - smtp + b.RLock() + found := strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || + strings.Contains(r.Responses[i].Data.RequestDecoded, s) || + strings.Contains(r.Responses[i].Data.MessageDecoded, s) + b.RUnlock() + + if found { + b.Lock() + r.Responses = append(r.Responses[:i], r.Responses[i+1:]...) + b.Unlock() + return true + } + } + } + return false +} diff --git a/v2/pkg/collaborator/collaborator.go b/v2/pkg/collaborator/collaborator.go deleted file mode 100644 index 4b6383b23..000000000 --- a/v2/pkg/collaborator/collaborator.go +++ /dev/null @@ -1,64 +0,0 @@ -package collaborator - -import ( - "strings" - "sync" - "time" - - "github.com/projectdiscovery/collaborator" -) - -const ( - PollSeconds = 5 - DefaultMaxBufferLimit = 150 -) - -var DefaultPollInterval time.Duration = time.Second * time.Duration(PollSeconds) - -var DefaultCollaborator BurpCollaborator = BurpCollaborator{Collab: collaborator.NewBurpCollaborator()} - -type BurpCollaborator struct { - sync.RWMutex - options *Options // unused - Collab *collaborator.BurpCollaborator -} - -type Options struct { - BIID string - PollInterval time.Duration - MaxBufferLimit int -} - -func New(options *Options) *BurpCollaborator { - collab := collaborator.NewBurpCollaborator() - collab.AddBIID(options.BIID) - collab.MaxBufferLimit = options.MaxBufferLimit - return &BurpCollaborator{Collab: collab, options: options} -} - -func (b *BurpCollaborator) Poll() { - // if no valid biids were provided just return - if len(b.Collab.BIIDs) > 0 { - go b.Collab.PollEach(DefaultPollInterval) - } -} - -func (b *BurpCollaborator) Has(s string) (found bool) { - foundAt := 0 - for _, r := range b.Collab.RespBuffer { - for i := 0; i < len(r.Responses); i++ { - // search in dns - http - smtp - b.RLock() - found = strings.Contains(r.Responses[i].Data.RawRequestDecoded, s) || strings.Contains(r.Responses[i].Data.RequestDecoded, s) || strings.Contains(r.Responses[i].Data.MessageDecoded, s) - b.RUnlock() - if found { - b.Lock() - r.Responses = removeMatch(r.Responses, foundAt) - b.Unlock() - break - } - } - } - - return -} diff --git a/v2/pkg/collaborator/util.go b/v2/pkg/collaborator/util.go deleted file mode 100644 index a6e35675b..000000000 --- a/v2/pkg/collaborator/util.go +++ /dev/null @@ -1,9 +0,0 @@ -package collaborator - -import ( - "github.com/projectdiscovery/collaborator" -) - -func removeMatch(responses []collaborator.BurpResponse, index int) []collaborator.BurpResponse { - return append(responses[:index], responses[index+1:]...) -} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d02cfda64..b52b8b145 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1 +1,65 @@ package http + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" +) + +// Request contains a http request to be made from a template +type Request struct { + // Path contains the path/s for the request + Path []string `yaml:"path"` + // Matchers contains the detection mechanism for the request to identify + // whether the request was successful + Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` + // Extractors contains the extraction mechanism for the request to identify + // and extract parts of the response. + Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` + // Raw contains raw requests + Raw []string `yaml:"raw,omitempty"` + Name string `yaml:"Name,omitempty"` + // AttackType is the attack type + // Sniper, PitchFork and ClusterBomb. Default is Sniper + AttackType string `yaml:"attack,omitempty"` + // Method is the request method, whether GET, POST, PUT, etc + Method string `yaml:"method"` + // Body is an optional parameter which contains the request body for POST methods, etc + Body string `yaml:"body,omitempty"` + // MatchersCondition is the condition of the matchers + // whether to use AND or OR. Default is OR. + MatchersCondition string `yaml:"matchers-condition,omitempty"` + // attackType is internal attack type + attackType generators.Type + // Path contains the path/s for the request variables + Payloads map[string]interface{} `yaml:"payloads,omitempty"` + // Headers contains headers to send with the request + Headers map[string]string `yaml:"headers,omitempty"` + // matchersCondition is internal condition for the matchers. + matchersCondition matchers.ConditionType + // MaxRedirects is the maximum number of redirects that should be followed. + MaxRedirects int `yaml:"max-redirects,omitempty"` + PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"` + PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"` + Threads int `yaml:"threads,omitempty"` + // Internal Finite State Machine keeping track of scan process + gsfm *GeneratorFSM + // CookieReuse is an optional setting that makes cookies shared within requests + CookieReuse bool `yaml:"cookie-reuse,omitempty"` + // Redirects specifies whether redirects should be followed. + Redirects bool `yaml:"redirects,omitempty"` + // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests) + // All requests must be indempotent (GET/POST) + Pipeline bool `yaml:"pipeline,omitempty"` + // Specify in order to skip request RFC normalization + Unsafe bool `yaml:"unsafe,omitempty"` + // DisableAutoHostname Enable/Disable Host header for unsafe raw requests + DisableAutoHostname bool `yaml:"disable-automatic-host-header,omitempty"` + // DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests + DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header,omitempty"` + // Race determines if all the request have to be attempted at the same time + // The minimum number fof requests is determined by threads + Race bool `yaml:"race,omitempty"` + // Number of same request to send in race condition attack + RaceNumberRequests int `yaml:"race_count,omitempty"` +} diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 1297fd9e2..974e793bd 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -13,7 +13,6 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/generators" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser" @@ -28,64 +27,6 @@ const ( var urlWithPortRgx = regexp.MustCompile(`{{BaseURL}}:(\d+)`) -// BulkHTTPRequest contains a request to be made from a template -type BulkHTTPRequest struct { - // Path contains the path/s for the request - Path []string `yaml:"path"` - // Matchers contains the detection mechanism for the request to identify - // whether the request was successful - Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` - // Extractors contains the extraction mechanism for the request to identify - // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` - // Raw contains raw requests - Raw []string `yaml:"raw,omitempty"` - Name string `yaml:"Name,omitempty"` - // AttackType is the attack type - // Sniper, PitchFork and ClusterBomb. Default is Sniper - AttackType string `yaml:"attack,omitempty"` - // Method is the request method, whether GET, POST, PUT, etc - Method string `yaml:"method"` - // Body is an optional parameter which contains the request body for POST methods, etc - Body string `yaml:"body,omitempty"` - // MatchersCondition is the condition of the matchers - // whether to use AND or OR. Default is OR. - MatchersCondition string `yaml:"matchers-condition,omitempty"` - // attackType is internal attack type - attackType generators.Type - // Path contains the path/s for the request variables - Payloads map[string]interface{} `yaml:"payloads,omitempty"` - // Headers contains headers to send with the request - Headers map[string]string `yaml:"headers,omitempty"` - // matchersCondition is internal condition for the matchers. - matchersCondition matchers.ConditionType - // MaxRedirects is the maximum number of redirects that should be followed. - MaxRedirects int `yaml:"max-redirects,omitempty"` - PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"` - PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"` - Threads int `yaml:"threads,omitempty"` - // Internal Finite State Machine keeping track of scan process - gsfm *GeneratorFSM - // CookieReuse is an optional setting that makes cookies shared within requests - CookieReuse bool `yaml:"cookie-reuse,omitempty"` - // Redirects specifies whether redirects should be followed. - Redirects bool `yaml:"redirects,omitempty"` - // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests) - // All requests must be indempotent (GET/POST) - Pipeline bool `yaml:"pipeline,omitempty"` - // Specify in order to skip request RFC normalization - Unsafe bool `yaml:"unsafe,omitempty"` - // DisableAutoHostname Enable/Disable Host header for unsafe raw requests - DisableAutoHostname bool `yaml:"disable-automatic-host-header,omitempty"` - // DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests - DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header,omitempty"` - // Race determines if all the request have to be attempted at the same time - // The minimum number fof requests is determined by threads - Race bool `yaml:"race,omitempty"` - // Number of same request to send in race condition attack - RaceNumberRequests int `yaml:"race_count,omitempty"` -} - // GetMatchersCondition returns the condition for the matcher func (r *BulkHTTPRequest) GetMatchersCondition() matchers.ConditionType { return r.matchersCondition From 2317e1ba1bcb80614ceb6ee39387eb91b28a6deb Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 23 Dec 2020 16:16:16 +0530 Subject: [PATCH 11/92] Data modelling, work on executor started --- v2/pkg/operators/common/dsl/dsl.go | 2 +- v2/pkg/operators/matchers/compile.go | 10 ++--- v2/pkg/operators/matchers/match.go | 4 +- v2/pkg/output/output.go | 13 ++++++ v2/pkg/protocols/http/http.go | 65 ++++++++++++++-------------- v2/pkg/protocols/protocols.go | 20 +++++++-- 6 files changed, 69 insertions(+), 45 deletions(-) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 70e69ac46..aa485b1cb 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -17,7 +17,7 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/collaborator" + "github.com/projectdiscovery/nuclei/v2/internal/collaborator" "github.com/spaolacci/murmur3" "github.com/spf13/cast" ) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index 73d17f9e9..e47210703 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -5,7 +5,7 @@ import ( "regexp" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" ) // CompileMatchers performs the initial setup operation on a matcher @@ -24,17 +24,15 @@ func (m *Matcher) CompileMatchers() error { if err != nil { return fmt.Errorf("could not compile regex: %s", regex) } - m.regexCompiled = append(m.regexCompiled, compiled) } // Compile the dsl expressions - for _, dsl := range m.DSL { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(dsl, generators.HelperFunctions()) + for _, expr := range m.DSL { + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) if err != nil { - return fmt.Errorf("could not compile dsl: %s", dsl) + return fmt.Errorf("could not compile dsl: %s", expr) } - m.dslCompiled = append(m.dslCompiled, compiled) } diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index f39f0d0ef..116654108 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -7,11 +7,11 @@ import ( "time" "github.com/miekg/dns" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) // Match matches a http response again a given matcher -func (m *Matcher) Match(resp *http.Response, body, headers string, duration time.Duration, data map[string]interface{}) bool { +func (m *Matcher) Match(resp *http.Response, body, headers string, duration time.Duration) bool { switch m.matcherType { case StatusMatcher: return m.isNegative(m.matchStatusCode(resp.StatusCode)) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index efaf410d4..c92e66ae3 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -41,6 +41,19 @@ const ( // Event is a single output structure from nuclei. type Event map[string]interface{} +// Error returns errors for the event if any +func (e Event) Error() error { + if data, ok := e["err"]; ok { + return data.(error) + } + return nil +} + +// SetError sets the error object for the event. +func (e Event) SetError(err error) { + e["err"] = err +} + // NewStandardWriter creates a new output writer based on user configurations func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) { colorizer := aurora.NewAurora(colors) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index b52b8b145..53138a212 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -8,42 +8,17 @@ import ( // Request contains a http request to be made from a template type Request struct { - // Path contains the path/s for the request - Path []string `yaml:"path"` - // Matchers contains the detection mechanism for the request to identify - // whether the request was successful - Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` - // Extractors contains the extraction mechanism for the request to identify - // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` - // Raw contains raw requests - Raw []string `yaml:"raw,omitempty"` - Name string `yaml:"Name,omitempty"` - // AttackType is the attack type - // Sniper, PitchFork and ClusterBomb. Default is Sniper - AttackType string `yaml:"attack,omitempty"` - // Method is the request method, whether GET, POST, PUT, etc - Method string `yaml:"method"` - // Body is an optional parameter which contains the request body for POST methods, etc - Body string `yaml:"body,omitempty"` - // MatchersCondition is the condition of the matchers - // whether to use AND or OR. Default is OR. - MatchersCondition string `yaml:"matchers-condition,omitempty"` - // attackType is internal attack type - attackType generators.Type - // Path contains the path/s for the request variables - Payloads map[string]interface{} `yaml:"payloads,omitempty"` - // Headers contains headers to send with the request - Headers map[string]string `yaml:"headers,omitempty"` - // matchersCondition is internal condition for the matchers. - matchersCondition matchers.ConditionType + // Number of same request to send in race condition attack + RaceNumberRequests int `yaml:"race_count,omitempty"` // MaxRedirects is the maximum number of redirects that should be followed. MaxRedirects int `yaml:"max-redirects,omitempty"` PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"` PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"` Threads int `yaml:"threads,omitempty"` - // Internal Finite State Machine keeping track of scan process - gsfm *GeneratorFSM + // attackType is internal attack type + attackType generators.Type + // matchersCondition is internal condition for the matchers. + matchersCondition matchers.ConditionType // CookieReuse is an optional setting that makes cookies shared within requests CookieReuse bool `yaml:"cookie-reuse,omitempty"` // Redirects specifies whether redirects should be followed. @@ -60,6 +35,30 @@ type Request struct { // Race determines if all the request have to be attempted at the same time // The minimum number fof requests is determined by threads Race bool `yaml:"race,omitempty"` - // Number of same request to send in race condition attack - RaceNumberRequests int `yaml:"race_count,omitempty"` + // Name is the name of the request + Name string `yaml:"Name,omitempty"` + // AttackType is the attack type + // Sniper, PitchFork and ClusterBomb. Default is Sniper + AttackType string `yaml:"attack,omitempty"` + // Method is the request method, whether GET, POST, PUT, etc + Method string `yaml:"method"` + // Body is an optional parameter which contains the request body for POST methods, etc + Body string `yaml:"body,omitempty"` + // MatchersCondition is the condition of the matchers + // whether to use AND or OR. Default is OR. + MatchersCondition string `yaml:"matchers-condition,omitempty"` + // Path contains the path/s for the request + Path []string `yaml:"path"` + // Raw contains raw requests + Raw []string `yaml:"raw,omitempty"` + // Matchers contains the detection mechanism for the request to identify + // whether the request was successful + Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` + // Extractors contains the extraction mechanism for the request to identify + // and extract parts of the response. + Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` + // Path contains the path/s for the request variables + Payloads map[string]interface{} `yaml:"payloads,omitempty"` + // Headers contains headers to send with the request + Headers map[string]string `yaml:"headers,omitempty"` } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 49576005d..05dd7a698 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,9 +1,23 @@ package protocols -// Rule is an interface implemented by a protocol rule -type Rule interface { - // Compile compiles the protocol request for further execution. +import "github.com/projectdiscovery/nuclei/v2/pkg/output" + +// RequestGenerator is an interface implemented by request generator for a protocol. +type RequestGenerator interface { + // Next returns the next request in queue for the generator interface. + // If no requests are remaining, next returns io.EOF error. + Next() (interface{}, error) + // Compile compiles the request generators preparing any requests possible. Compile() error // Requests returns the total number of requests the rule will perform Requests() int64 } + +// Executer executes requests from a generator and returns an output event. +type Executer interface { + // Execute executes the generator requests and returns an output event channel. + Execute(generator RequestGenerator, callback OutputEventCallback) error +} + +// OutputEventCallback is a callback for each recieved output from executor +type OutputEventCallback func(event output.Event) From 095902089f1703459b611ae3b6f8fa74e7589f52 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 23 Dec 2020 20:46:19 +0530 Subject: [PATCH 12/92] Added a http client pooling implementation --- .../protocols/http/clientpool/clientpool.go | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 v2/pkg/protocols/http/clientpool/clientpool.go diff --git a/v2/pkg/protocols/http/clientpool/clientpool.go b/v2/pkg/protocols/http/clientpool/clientpool.go new file mode 100644 index 000000000..747b55371 --- /dev/null +++ b/v2/pkg/protocols/http/clientpool/clientpool.go @@ -0,0 +1,171 @@ +package clientpool + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" + "golang.org/x/net/proxy" +) + +var ( + dialer *fastdialer.Dialer + poolMutex *sync.RWMutex + clientPool map[string]*retryablehttp.Client +) + +func init() { + poolMutex = &sync.RWMutex{} + clientPool = make(map[string]*retryablehttp.Client) +} + +// Configuration contains the custom configuration options for a client +type Configuration struct { + // Threads contains the threads for the client + Threads int + // MaxRedirects is the maximum number of redirects to follow + MaxRedirects int + // FollowRedirects specifies whether to follow redirects + FollowRedirects bool +} + +// Hash returns the hash of the configuration to allow client pooling +func (c *Configuration) Hash() string { + builder := &strings.Builder{} + builder.WriteString("t") + builder.WriteString(strconv.Itoa(c.Threads)) + builder.WriteString("m") + builder.WriteString(strconv.Itoa(c.MaxRedirects)) + builder.WriteString("f") + builder.WriteString(strconv.FormatBool(c.FollowRedirects)) + hash := builder.String() + return hash +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { + var proxyURL *url.URL + var err error + + if dialer == nil { + dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions) + } + if err != nil { + return nil, errors.Wrap(err, "could not create dialer") + } + + hash := configuration.Hash() + poolMutex.RLock() + if client, ok := clientPool[hash]; ok { + poolMutex.RUnlock() + return client, nil + } + poolMutex.RUnlock() + + if options.ProxyURL != "" { + proxyURL, err = url.Parse(options.ProxyURL) + } + if err != nil { + return nil, err + } + + // Multiple Host + retryablehttpOptions := retryablehttp.DefaultOptionsSpraying + disableKeepAlives := true + maxIdleConns := 0 + maxConnsPerHost := 0 + maxIdleConnsPerHost := -1 + + if configuration.Threads > 0 { + // Single host + retryablehttpOptions = retryablehttp.DefaultOptionsSingle + disableKeepAlives = false + maxIdleConnsPerHost = 500 + maxConnsPerHost = 500 + } + + retryablehttpOptions.RetryWaitMax = 10 * time.Second + retryablehttpOptions.RetryMax = options.Retries + followRedirects := configuration.FollowRedirects + maxRedirects := configuration.MaxRedirects + + transport := &http.Transport{ + DialContext: dialer.Dial, + MaxIdleConns: maxIdleConns, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + MaxConnsPerHost: maxConnsPerHost, + TLSClientConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + DisableKeepAlives: disableKeepAlives, + } + + // Attempts to overwrite the dial function with the socks proxied version + if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, err := url.Parse(options.ProxySocksURL) + if err == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if err == nil { + transport.DialContext = dc.DialContext + } + } + if proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + + client := retryablehttp.NewWithHTTPClient(&http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout) * time.Second, + CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), + }, retryablehttpOptions) + + poolMutex.Lock() + clientPool[hash] = client + poolMutex.Unlock() + return client, nil +} + +const defaultMaxRedirects = 10 + +type checkRedirectFunc func(req *http.Request, via []*http.Request) error + +func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc { + return func(req *http.Request, via []*http.Request) error { + if !followRedirects { + return http.ErrUseLastResponse + } + + if maxRedirects == 0 { + if len(via) > defaultMaxRedirects { + return http.ErrUseLastResponse + } + return nil + } + + if len(via) > maxRedirects { + return http.ErrUseLastResponse + } + return nil + } +} From c4428824b6679add43975070160cc779eb5cd263 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 23 Dec 2020 20:46:42 +0530 Subject: [PATCH 13/92] Misc work on restructuring + adding stuff --- v2/internal/runner/options.go | 100 ------------------------ v2/pkg/executer/executer_dns.go | 21 +---- v2/pkg/executer/executer_http.go | 129 ++----------------------------- v2/pkg/output/output.go | 19 ++--- v2/pkg/protocols/protocols.go | 31 ++++---- v2/pkg/types/types.go | 102 ++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 267 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 8f4519043..a7b66dee2 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,110 +5,10 @@ import ( "flag" "net/url" "os" - "strings" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" ) -// Options contains the configuration options for nuclei scanner. -type Options struct { - // Vhost marks the input specified as VHOST input - Vhost bool - // RandomAgent generates random User-Agent - RandomAgent bool - // Metrics enables display of metrics via an http endpoint - Metrics bool - // Sandbox mode allows users to run isolated workflows with system commands disabled - Sandbox bool - // Debug mode allows debugging request/responses for the engine - Debug bool - // Silent suppresses any extra text and only writes found URLs on screen. - Silent bool - // Version specifies if we should just show version and exit - Version bool - // Verbose flag indicates whether to show verbose output or not - Verbose bool - // No-Color disables the colored output. - NoColor bool - // UpdateTemplates updates the templates installed at startup - UpdateTemplates bool - // JSON writes json output to files - JSON bool - // JSONRequests writes requests/responses for matches in JSON output - JSONRequests bool - // EnableProgressBar enables progress bar - EnableProgressBar bool - // TemplatesVersion shows the templates installed version - TemplatesVersion bool - // TemplateList lists available templates - TemplateList bool - // Stdin specifies whether stdin input was given to the process - Stdin bool - // StopAtFirstMatch stops processing template at first full match (this may break chained requests) - StopAtFirstMatch bool - // NoMeta disables display of metadata for the matches - NoMeta bool - // Project is used to avoid sending same HTTP request multiple times - Project bool - // MetricsPort is the port to show metrics on - MetricsPort int - // MaxWorkflowDuration is the maximum time a workflow can run for a URL - MaxWorkflowDuration int - // BulkSize is the of targets analyzed in parallel for each template - BulkSize int - // TemplateThreads is the number of templates executed in parallel - TemplateThreads int - // Timeout is the seconds to wait for a response from the server. - Timeout int - // Retries is the number of times to retry the request - Retries int - // Rate-Limit is the maximum number of requests per specified target - RateLimit int - // Thread controls the number of concurrent requests to make. - Threads int - // BurpCollaboratorBiid is the Burp Collaborator BIID for polling interactions. - BurpCollaboratorBiid string - // ProjectPath allows nuclei to use a user defined project folder - ProjectPath string - // Severity filters templates based on their severity and only run the matching ones. - Severity string - // Target is a single URL/Domain to scan using a template - Target string - // Targets specifies the targets to scan using templates. - Targets string - // Output is the file to write found results to. - Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string - // TemplatesDirectory is the directory to use for storing templates - TemplatesDirectory string - // TraceLogFile specifies a file to write with the trace of all requests - TraceLogFile string - // Templates specifies the template/templates to use - Templates StringSlice - // ExcludedTemplates specifies the template/templates to exclude - ExcludedTemplates StringSlice - // CustomHeaders is the list of custom global headers to send with each request. - CustomHeaders requests.CustomHeaders -} - -// StringSlice is a slice of strings as input -type StringSlice []string - -// String returns the stringified version of string slice -func (s *StringSlice) String() string { - return strings.Join(*s, ",") -} - -// Set appends a value to the string slice -func (s *StringSlice) Set(value string) error { - *s = append(*s, value) - return nil -} - // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { options := &Options{} diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index ef5d33757..1a8623572 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -22,23 +22,10 @@ import ( // DNSExecuter is a client for performing a DNS request // for a template. type DNSExecuter struct { - // hm *hybrid.HybridMap // Unused - coloredOutput bool - debug bool - jsonOutput bool - jsonRequest bool - noMeta bool - Results bool - vhost bool - traceLog tracelog.Log - dnsClient *retryabledns.Client - template *templates.Template - dnsRequest *requests.DNSRequest - writer *bufwriter.Writer - ratelimiter ratelimit.Limiter - - colorizer colorizer.NucleiColorizer - decolorizer *regexp.Regexp + dnsClient *retryabledns.Client + template *templates.Template + dnsRequest *requests.DNSRequest + ratelimiter ratelimit.Limiter } // DefaultResolvers contains the list of resolvers known to be trusted. diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index e23dcddd3..a43cf82e3 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -2,12 +2,9 @@ package executer import ( "bytes" - "context" - "crypto/tls" "fmt" "io" "io/ioutil" - "net" "net/http" "net/http/cookiejar" "net/http/httputil" @@ -36,7 +33,6 @@ import ( "github.com/projectdiscovery/retryablehttp-go" "github.com/remeh/sizedwaitgroup" "go.uber.org/ratelimit" - "golang.org/x/net/proxy" ) const ( @@ -74,30 +70,12 @@ type HTTPExecuter struct { // HTTPOptions contains configuration options for the HTTP executer. type HTTPOptions struct { - RandomAgent bool - Debug bool - JSON bool - JSONRequests bool - NoMeta bool - CookieReuse bool - ColoredOutput bool - StopAtFirstMatch bool - Vhost bool - Timeout int - Retries int - ProxyURL string - ProxySocksURL string - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - Writer *bufwriter.Writer - CustomHeaders requests.CustomHeaders - CookieJar *cookiejar.Jar - Colorizer *colorizer.NucleiColorizer - Decolorizer *regexp.Regexp - TraceLog tracelog.Log - PF *projetctfile.ProjectFile - RateLimiter ratelimit.Limiter - Dialer *fastdialer.Dialer + Template *templates.Template + BulkHTTPRequest *requests.BulkHTTPRequest + CookieJar *cookiejar.Jar + PF *projetctfile.ProjectFile + RateLimiter ratelimit.Limiter + Dialer *fastdialer.Dialer } // NewHTTPExecuter creates a new HTTP executer from a template @@ -108,10 +86,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { err error ) - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) - } - if err != nil { return nil, err } @@ -654,97 +628,6 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, // Close closes the http executer for a template. func (e *HTTPExecuter) Close() {} -// makeHTTPClient creates a http client -func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client { - // Multiple Host - retryablehttpOptions := retryablehttp.DefaultOptionsSpraying - disableKeepAlives := true - maxIdleConns := 0 - maxConnsPerHost := 0 - maxIdleConnsPerHost := -1 - - if options.BulkHTTPRequest.Threads > 0 { - // Single host - retryablehttpOptions = retryablehttp.DefaultOptionsSingle - disableKeepAlives = false - maxIdleConnsPerHost = 500 - maxConnsPerHost = 500 - } - - retryablehttpOptions.RetryWaitMax = 10 * time.Second - retryablehttpOptions.RetryMax = options.Retries - followRedirects := options.BulkHTTPRequest.Redirects - maxRedirects := options.BulkHTTPRequest.MaxRedirects - - transport := &http.Transport{ - DialContext: options.Dialer.Dial, - MaxIdleConns: maxIdleConns, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - MaxConnsPerHost: maxConnsPerHost, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - DisableKeepAlives: disableKeepAlives, - } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, err := url.Parse(options.ProxySocksURL) - - if err == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - - dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - - if err == nil { - transport.DialContext = dc.DialContext - } - } - - if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) - } - - return retryablehttp.NewWithHTTPClient(&http.Client{ - Transport: transport, - Timeout: time.Duration(options.Timeout) * time.Second, - CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), - }, retryablehttpOptions) -} - -type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error - -func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc { - return func(_ *http.Request, requests []*http.Request) error { - if !followRedirects { - return http.ErrUseLastResponse - } - - if maxRedirects == 0 { - if len(requests) > ten { - return http.ErrUseLastResponse - } - - return nil - } - - if len(requests) > maxRedirects { - return http.ErrUseLastResponse - } - - return nil - } -} - func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { for _, customHeader := range e.customHeaders { // This should be pre-computed somewhere and done only once diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index c92e66ae3..60de992c1 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -2,6 +2,7 @@ package output import ( "os" + "regexp" "sync" jsoniter "github.com/json-iterator/go" @@ -38,22 +39,11 @@ const ( undefined string = "undefined" ) +var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) + // Event is a single output structure from nuclei. type Event map[string]interface{} -// Error returns errors for the event if any -func (e Event) Error() error { - if data, ok := e["err"]; ok { - return data.(error) - } - return nil -} - -// SetError sets the error object for the event. -func (e Event) SetError(err error) { - e["err"] = err -} - // NewStandardWriter creates a new output writer based on user configurations func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) { colorizer := aurora.NewAurora(colors) @@ -109,6 +99,9 @@ func (w *StandardWriter) Write(event Event) error { } _, _ = os.Stdout.Write(data) if w.outputFile != nil { + if !w.json { + data = decolorizerRegex.ReplaceAll(data, []byte("")) + } if writeErr := w.outputFile.Write(data); writeErr != nil { return errors.Wrap(err, "could not write to output") } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 05dd7a698..7e58eb645 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,23 +1,26 @@ package protocols -import "github.com/projectdiscovery/nuclei/v2/pkg/output" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) -// RequestGenerator is an interface implemented by request generator for a protocol. -type RequestGenerator interface { - // Next returns the next request in queue for the generator interface. - // If no requests are remaining, next returns io.EOF error. - Next() (interface{}, error) +// Executer is an interface implemented any protocol based request generator. +type Executer interface { // Compile compiles the request generators preparing any requests possible. - Compile() error + Compile(options ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int64 + // Execute executes the protocol requests and returns an output event channel. + Execute(input string) (bool, error) + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. + ExecuteWithResults(input string) ([]output.Event, error) } -// Executer executes requests from a generator and returns an output event. -type Executer interface { - // Execute executes the generator requests and returns an output event channel. - Execute(generator RequestGenerator, callback OutputEventCallback) error +// ExecuterOptions contains the configuration options for executer clients +type ExecuterOptions struct { + // Output is a writer interface for writing output events from executer. + Output output.Writer + // Options contains configuration options for the executer + Options *types.Options } - -// OutputEventCallback is a callback for each recieved output from executor -type OutputEventCallback func(event output.Event) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index ab1254f4c..a046f73b9 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -1 +1,103 @@ package types + +import ( + "strings" +) + +// Options contains the configuration options for nuclei scanner. +type Options struct { + // Vhost marks the input specified as VHOST input + Vhost bool + // RandomAgent generates random User-Agent + RandomAgent bool + // Metrics enables display of metrics via an http endpoint + Metrics bool + // Sandbox mode allows users to run isolated workflows with system commands disabled + Sandbox bool + // Debug mode allows debugging request/responses for the engine + Debug bool + // Silent suppresses any extra text and only writes found URLs on screen. + Silent bool + // Version specifies if we should just show version and exit + Version bool + // Verbose flag indicates whether to show verbose output or not + Verbose bool + // No-Color disables the colored output. + NoColor bool + // UpdateTemplates updates the templates installed at startup + UpdateTemplates bool + // JSON writes json output to files + JSON bool + // JSONRequests writes requests/responses for matches in JSON output + JSONRequests bool + // EnableProgressBar enables progress bar + EnableProgressBar bool + // TemplatesVersion shows the templates installed version + TemplatesVersion bool + // TemplateList lists available templates + TemplateList bool + // Stdin specifies whether stdin input was given to the process + Stdin bool + // StopAtFirstMatch stops processing template at first full match (this may break chained requests) + StopAtFirstMatch bool + // NoMeta disables display of metadata for the matches + NoMeta bool + // Project is used to avoid sending same HTTP request multiple times + Project bool + // MetricsPort is the port to show metrics on + MetricsPort int + // MaxWorkflowDuration is the maximum time a workflow can run for a URL + MaxWorkflowDuration int + // BulkSize is the of targets analyzed in parallel for each template + BulkSize int + // TemplateThreads is the number of templates executed in parallel + TemplateThreads int + // Timeout is the seconds to wait for a response from the server. + Timeout int + // Retries is the number of times to retry the request + Retries int + // Rate-Limit is the maximum number of requests per specified target + RateLimit int + // Thread controls the number of concurrent requests to make. + Threads int + // BurpCollaboratorBiid is the Burp Collaborator BIID for polling interactions. + BurpCollaboratorBiid string + // ProjectPath allows nuclei to use a user defined project folder + ProjectPath string + // Severity filters templates based on their severity and only run the matching ones. + Severity string + // Target is a single URL/Domain to scan using a template + Target string + // Targets specifies the targets to scan using templates. + Targets string + // Output is the file to write found results to. + Output string + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string + // TemplatesDirectory is the directory to use for storing templates + TemplatesDirectory string + // TraceLogFile specifies a file to write with the trace of all requests + TraceLogFile string + // Templates specifies the template/templates to use + Templates StringSlice + // ExcludedTemplates specifies the template/templates to exclude + ExcludedTemplates StringSlice + // CustomHeaders is the list of custom global headers to send with each request. + CustomHeaders StringSlice +} + +// StringSlice is a slice of strings as input +type StringSlice []string + +// String returns the stringified version of string slice +func (s *StringSlice) String() string { + return strings.Join(*s, ",") +} + +// Set appends a value to the string slice +func (s *StringSlice) Set(value string) error { + *s = append(*s, value) + return nil +} From ff4c61a0eb9455bb54cfb783bb9c8e22c4da66da Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 23 Dec 2020 22:09:11 +0530 Subject: [PATCH 14/92] Added dns client pool + misc changes to http client pool --- v2/pkg/protocols/dns/clientpool/clientpool.go | 78 +++++++++++++++++++ .../protocols/http/clientpool/clientpool.go | 36 ++++++++- 2 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/protocols/dns/clientpool/clientpool.go diff --git a/v2/pkg/protocols/dns/clientpool/clientpool.go b/v2/pkg/protocols/dns/clientpool/clientpool.go new file mode 100644 index 000000000..442dc8e5e --- /dev/null +++ b/v2/pkg/protocols/dns/clientpool/clientpool.go @@ -0,0 +1,78 @@ +package clientpool + +import ( + "strconv" + "strings" + "sync" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/retryabledns" +) + +var ( + poolMutex *sync.RWMutex + normalClient *retryabledns.Client + clientPool map[string]*retryabledns.Client +) + +// defaultResolvers contains the list of resolvers known to be trusted. +var defaultResolvers = []string{ + "1.1.1.1:53", // Cloudflare + "1.0.0.1:53", // Cloudflare + "8.8.8.8:53", // Google + "8.8.4.4:53", // Google +} + +// Init initializes the clientpool implementation +func Init(options *types.Options) error { + // Don't create clients if already created in past. + if normalClient != nil { + return nil + } + poolMutex = &sync.RWMutex{} + clientPool = make(map[string]*retryabledns.Client) + + if client, err := Get(options, &Configuration{}); err != nil { + return err + } else { + normalClient = client + } + return nil +} + +// Configuration contains the custom configuration options for a client +type Configuration struct { + // Retries contains the retries for the dns client + Retries int +} + +// Hash returns the hash of the configuration to allow client pooling +func (c *Configuration) Hash() string { + builder := &strings.Builder{} + builder.Grow(8) + builder.WriteString("r") + builder.WriteString(strconv.Itoa(c.Retries)) + hash := builder.String() + return hash +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { + if !(configuration.Retries > 0) { + return normalClient, nil + } + hash := configuration.Hash() + poolMutex.RLock() + if client, ok := clientPool[hash]; ok { + poolMutex.RUnlock() + return client, nil + } + poolMutex.RUnlock() + + client := retryabledns.New(defaultResolvers, configuration.Retries) + + poolMutex.Lock() + clientPool[hash] = client + poolMutex.Unlock() + return client, nil +} diff --git a/v2/pkg/protocols/http/clientpool/clientpool.go b/v2/pkg/protocols/http/clientpool/clientpool.go index 747b55371..90bc2a89a 100644 --- a/v2/pkg/protocols/http/clientpool/clientpool.go +++ b/v2/pkg/protocols/http/clientpool/clientpool.go @@ -15,19 +15,34 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" "golang.org/x/net/proxy" ) var ( - dialer *fastdialer.Dialer - poolMutex *sync.RWMutex - clientPool map[string]*retryablehttp.Client + dialer *fastdialer.Dialer + rawhttpClient *rawhttp.Client + poolMutex *sync.RWMutex + normalClient *retryablehttp.Client + clientPool map[string]*retryablehttp.Client ) -func init() { +// Init initializes the clientpool implementation +func Init(options *types.Options) error { + // Don't create clients if already created in past. + if normalClient != nil { + return nil + } poolMutex = &sync.RWMutex{} clientPool = make(map[string]*retryablehttp.Client) + + if client, err := Get(options, &Configuration{}); err != nil { + return err + } else { + normalClient = client + } + return nil } // Configuration contains the custom configuration options for a client @@ -43,6 +58,7 @@ type Configuration struct { // Hash returns the hash of the configuration to allow client pooling func (c *Configuration) Hash() string { builder := &strings.Builder{} + builder.Grow(16) builder.WriteString("t") builder.WriteString(strconv.Itoa(c.Threads)) builder.WriteString("m") @@ -53,8 +69,19 @@ func (c *Configuration) Hash() string { return hash } +// GetRawHTTP returns the rawhttp request client +func GetRawHTTP() *rawhttp.Client { + if rawhttpClient == nil { + rawhttpClient = rawhttp.NewClient(rawhttp.DefaultOptions) + } + return rawhttpClient +} + // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { + if !(configuration.Threads > 0 && configuration.MaxRedirects > 0 && configuration.FollowRedirects) { + return normalClient, nil + } var proxyURL *url.URL var err error @@ -139,6 +166,7 @@ func Get(options *types.Options, configuration *Configuration) (*retryablehttp.C Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), }, retryablehttpOptions) + client.CheckRetry = retryablehttp.HostSprayRetryPolicy() poolMutex.Lock() clientPool[hash] = client From 8a64578890ea30a875631e171d2e9174e1eb8219 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 24 Dec 2020 01:41:32 +0530 Subject: [PATCH 15/92] Work on operators package and generic protocol agnostic matching capabilities --- v2/pkg/operators/matchers/compile.go | 4 ++ v2/pkg/operators/matchers/match.go | 74 ++++++--------------------- v2/pkg/operators/matchers/matchers.go | 1 - v2/pkg/operators/operators.go | 1 - 4 files changed, 20 insertions(+), 60 deletions(-) diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index e47210703..abf862140 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -17,6 +17,10 @@ func (m *Matcher) CompileMatchers() error { if !ok { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } + // By default, match on all if user hasn't provided any specific items + if m.Part == "" { + m.Part = "all" + } // Compile the regexes for _, regex := range m.Regex { diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 116654108..19ba7d29d 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -2,75 +2,34 @@ package matchers import ( "encoding/hex" - "net/http" "strings" - "time" - - "github.com/miekg/dns" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) -// Match matches a http response again a given matcher -func (m *Matcher) Match(resp *http.Response, body, headers string, duration time.Duration) bool { - switch m.matcherType { - case StatusMatcher: - return m.isNegative(m.matchStatusCode(resp.StatusCode)) - case SizeMatcher: - return m.isNegative(m.matchSizeCode(len(body))) - case WordsMatcher: - // Match the parts as required for word check - if m.Part == "body" { - return m.isNegative(m.matchWords(body)) - } else if m.Part == "header" { - return m.isNegative(m.matchWords(headers)) - } else { - return m.isNegative(m.matchWords(headers) || m.matchWords(body)) - } - case RegexMatcher: - // Match the parts as required for regex check - if m.Part == "body" { - return m.isNegative(m.matchRegex(body)) - } else if m.Part == "header" { - return m.isNegative(m.matchRegex(headers)) - } else { - return m.isNegative(m.matchRegex(headers) || m.matchRegex(body)) - } - case BinaryMatcher: - // Match the parts as required for binary characters check - if m.Part == "body" { - return m.isNegative(m.matchBinary(body)) - } else if m.Part == "header" { - return m.isNegative(m.matchBinary(headers)) - } else { - return m.isNegative(m.matchBinary(headers) || m.matchBinary(body)) - } - case DSLMatcher: - // Match complex query - return m.isNegative(m.matchDSL(generators.MergeMaps(HTTPToMap(resp, body, headers, duration, ""), data))) +// Match matches a generic data response again a given matcher +func (m *Matcher) Match(data map[string]interface{}) bool { + part, ok := data[m.Part] + if !ok { + return false } + partString := part.(string) - return false -} - -// MatchDNS matches a dns response against a given matcher -func (m *Matcher) MatchDNS(msg *dns.Msg) bool { switch m.matcherType { case StatusMatcher: - return m.isNegative(m.matchStatusCode(msg.Rcode)) + statusCode, ok := data["status_code"] + if !ok { + return false + } + return m.isNegative(m.matchStatusCode(statusCode.(int))) case SizeMatcher: - return m.matchSizeCode(msg.Len()) + return m.isNegative(m.matchSizeCode(len(partString))) case WordsMatcher: - // Match for word check - return m.matchWords(msg.String()) + return m.isNegative(m.matchWords(partString)) case RegexMatcher: - // Match regex check - return m.matchRegex(msg.String()) + return m.isNegative(m.matchRegex(partString)) case BinaryMatcher: - // Match binary characters check - return m.matchBinary(msg.String()) + return m.isNegative(m.matchBinary(partString)) case DSLMatcher: - // Match complex query - return m.matchDSL(DNSToMap(msg, "")) + return m.isNegative(m.matchDSL(data)) } return false } @@ -88,7 +47,6 @@ func (m *Matcher) matchStatusCode(statusCode int) bool { // Return on the first match. return true } - return false } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index c825e5e16..930461beb 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -94,6 +94,5 @@ func (m *Matcher) isNegative(data bool) bool { if m.Negative { return !data } - return data } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 830830e98..c60e153a1 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -16,7 +16,6 @@ type Operators struct { // MatchersCondition is the condition of the matchers // whether to use AND or OR. Default is OR. MatchersCondition string `yaml:"matchers-condition"` - // cached variables that may be used along with request. matchersCondition matchers.ConditionType } From 10642c6c77bc1ffa76c32d0d41c07b4b1d0074f7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 24 Dec 2020 01:42:04 +0530 Subject: [PATCH 16/92] Misc work on protocols --- v2/pkg/protocols/dns/matchers.go | 3 +- v2/pkg/protocols/http/http.go | 82 +++++++++++++------------------ v2/pkg/protocols/http/matchers.go | 4 +- v2/pkg/protocols/protocols.go | 5 +- 4 files changed, 42 insertions(+), 52 deletions(-) diff --git a/v2/pkg/protocols/dns/matchers.go b/v2/pkg/protocols/dns/matchers.go index 371f3576d..1e32bfffa 100644 --- a/v2/pkg/protocols/dns/matchers.go +++ b/v2/pkg/protocols/dns/matchers.go @@ -10,8 +10,6 @@ import ( func responseToDSLMap(msg *dns.Msg) map[string]interface{} { data := make(map[string]interface{}, 6) - data["rcode"] = msg.Rcode - buffer := &bytes.Buffer{} for _, question := range msg.Question { buffer.WriteString(question.String()) @@ -38,5 +36,6 @@ func responseToDSLMap(msg *dns.Msg) map[string]interface{} { buffer.Reset() data["raw"] = msg.String() + data["status_code"] = msg.Rcode return data } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 53138a212..3ab61eaf8 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,64 +1,52 @@ package http -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" -) +import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" // Request contains a http request to be made from a template type Request struct { - // Number of same request to send in race condition attack - RaceNumberRequests int `yaml:"race_count,omitempty"` - // MaxRedirects is the maximum number of redirects that should be followed. - MaxRedirects int `yaml:"max-redirects,omitempty"` - PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections,omitempty"` - PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection,omitempty"` - Threads int `yaml:"threads,omitempty"` - // attackType is internal attack type - attackType generators.Type - // matchersCondition is internal condition for the matchers. - matchersCondition matchers.ConditionType - // CookieReuse is an optional setting that makes cookies shared within requests - CookieReuse bool `yaml:"cookie-reuse,omitempty"` - // Redirects specifies whether redirects should be followed. - Redirects bool `yaml:"redirects,omitempty"` - // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests) - // All requests must be indempotent (GET/POST) - Pipeline bool `yaml:"pipeline,omitempty"` - // Specify in order to skip request RFC normalization - Unsafe bool `yaml:"unsafe,omitempty"` - // DisableAutoHostname Enable/Disable Host header for unsafe raw requests - DisableAutoHostname bool `yaml:"disable-automatic-host-header,omitempty"` - // DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests - DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header,omitempty"` - // Race determines if all the request have to be attempted at the same time - // The minimum number fof requests is determined by threads - Race bool `yaml:"race,omitempty"` // Name is the name of the request - Name string `yaml:"Name,omitempty"` + Name string `yaml:"Name"` // AttackType is the attack type // Sniper, PitchFork and ClusterBomb. Default is Sniper - AttackType string `yaml:"attack,omitempty"` + AttackType string `yaml:"attack"` // Method is the request method, whether GET, POST, PUT, etc Method string `yaml:"method"` // Body is an optional parameter which contains the request body for POST methods, etc - Body string `yaml:"body,omitempty"` - // MatchersCondition is the condition of the matchers - // whether to use AND or OR. Default is OR. - MatchersCondition string `yaml:"matchers-condition,omitempty"` + Body string `yaml:"body"` // Path contains the path/s for the request Path []string `yaml:"path"` // Raw contains raw requests - Raw []string `yaml:"raw,omitempty"` - // Matchers contains the detection mechanism for the request to identify - // whether the request was successful - Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` - // Extractors contains the extraction mechanism for the request to identify - // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` + Raw []string `yaml:"raw"` // Path contains the path/s for the request variables - Payloads map[string]interface{} `yaml:"payloads,omitempty"` + Payloads map[string]interface{} `yaml:"payloads"` // Headers contains headers to send with the request - Headers map[string]string `yaml:"headers,omitempty"` + Headers map[string]string `yaml:"headers"` + // RaceNumberRequests is the number of same request to send in race condition attack + RaceNumberRequests int `yaml:"race_count"` + // MaxRedirects is the maximum number of redirects that should be followed. + MaxRedirects int `yaml:"max-redirects"` + // PipelineConcurrentConnections is number of connections in pipelining + PipelineConcurrentConnections int `yaml:"pipeline-concurrent-connections"` + // PipelineRequestsPerConnection is number of requests in pipelining + PipelineRequestsPerConnection int `yaml:"pipeline-requests-per-connection"` + // Threads specifies number of threads for sending requests + Threads int `yaml:"threads"` + // CookieReuse is an optional setting that makes cookies shared within requests + CookieReuse bool `yaml:"cookie-reuse"` + // Redirects specifies whether redirects should be followed. + Redirects bool `yaml:"redirects"` + // Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining (race conditions/billions requests) + // All requests must be indempotent (GET/POST) + Pipeline bool `yaml:"pipeline"` + // Specify in order to skip request RFC normalization + Unsafe bool `yaml:"unsafe"` + // DisableAutoHostname Enable/Disable Host header for unsafe raw requests + DisableAutoHostname bool `yaml:"disable-automatic-host-header"` + // DisableAutoContentLength Enable/Disable Content-Length header for unsafe raw requests + DisableAutoContentLength bool `yaml:"disable-automatic-content-length-header"` + // Race determines if all the request have to be attempted at the same time + // The minimum number fof requests is determined by threads + Race bool `yaml:"race"` + + attackType generators.Type } diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/matchers.go index 8b2a0a350..6dcfb62ab 100644 --- a/v2/pkg/protocols/http/matchers.go +++ b/v2/pkg/protocols/http/matchers.go @@ -17,12 +17,12 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode + data["body"] = body for k, v := range resp.Header { k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) data[k] = strings.Join(v, " ") } - data["all_headers"] = headers - data["body"] = body + data["headers"] = headers if r, err := httputil.DumpResponse(resp, true); err == nil { data["raw"] = string(r) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 7e58eb645..09542784b 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -3,6 +3,7 @@ package protocols import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "go.uber.org/ratelimit" ) // Executer is an interface implemented any protocol based request generator. @@ -21,6 +22,8 @@ type Executer interface { type ExecuterOptions struct { // Output is a writer interface for writing output events from executer. Output output.Writer - // Options contains configuration options for the executer + // Options contains configuration options for the executer. Options *types.Options + // RateLimiter is a rate-limiter for limiting sent number of requests. + RateLimiter ratelimit.Limiter } From 5153647e0fb36e16f6160f6ba0e97968418d75d3 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 24 Dec 2020 12:13:18 +0530 Subject: [PATCH 17/92] Misc work on extractors + compat --- v2/pkg/executer/executer_dns.go | 36 +-------- v2/pkg/executer/executer_http.go | 18 ----- v2/pkg/operators/common/dsl/dsl.go | 76 +++++++++---------- v2/pkg/operators/extractors/compile.go | 10 +-- v2/pkg/operators/extractors/extract.go | 90 +++++++---------------- v2/pkg/operators/extractors/extractors.go | 26 ------- v2/pkg/operators/matchers/compile.go | 4 +- v2/pkg/output/format_screen.go | 8 +- v2/pkg/protocols/dns/matchers.go | 4 +- v2/pkg/protocols/http/matchers.go | 12 ++- v2/pkg/requests/bulk-http-request.go | 15 ---- v2/pkg/types/interfaces.go | 89 ++++++++++++++++++++++ 12 files changed, 177 insertions(+), 211 deletions(-) create mode 100644 v2/pkg/types/interfaces.go diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 1a8623572..86d5b4e0d 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -3,61 +3,31 @@ package executer import ( "fmt" "os" - "regexp" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/internal/tracelog" - "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - retryabledns "github.com/projectdiscovery/retryabledns" - "go.uber.org/ratelimit" ) // DNSExecuter is a client for performing a DNS request // for a template. type DNSExecuter struct { - dnsClient *retryabledns.Client - template *templates.Template - dnsRequest *requests.DNSRequest - ratelimiter ratelimit.Limiter -} - -// DefaultResolvers contains the list of resolvers known to be trusted. -var DefaultResolvers = []string{ - "1.1.1.1:53", // Cloudflare - "1.0.0.1:53", // Cloudflare - "8.8.8.8:53", // Google - "8.8.4.4:53", // Google + template *templates.Template } // DNSOptions contains configuration options for the DNS executer. type DNSOptions struct { - ColoredOutput bool - Debug bool - JSON bool - JSONRequests bool - NoMeta bool - VHost bool - TraceLog tracelog.Log - Template *templates.Template - DNSRequest *requests.DNSRequest - Writer *bufwriter.Writer - - Colorizer colorizer.NucleiColorizer - Decolorizer *regexp.Regexp - RateLimiter ratelimit.Limiter + Template *templates.Template + DNSRequest *requests.DNSRequest } // NewDNSExecuter creates a new DNS executer from a template // and a DNS request query. func NewDNSExecuter(options *DNSOptions) *DNSExecuter { - dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries) executer := &DNSExecuter{ debug: options.Debug, diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index a43cf82e3..89d0959f1 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -18,7 +18,6 @@ import ( "github.com/corpix/uarand" "github.com/pkg/errors" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" "github.com/projectdiscovery/nuclei/v2/internal/progress" @@ -74,8 +73,6 @@ type HTTPOptions struct { BulkHTTPRequest *requests.BulkHTTPRequest CookieJar *cookiejar.Jar PF *projetctfile.ProjectFile - RateLimiter ratelimit.Limiter - Dialer *fastdialer.Dialer } // NewHTTPExecuter creates a new HTTP executer from a template @@ -93,7 +90,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { // Create the HTTP Client client := makeHTTPClient(proxyURL, options) // nolint:bodyclose // false positive there is no body to close yet - client.CheckRetry = retryablehttp.HostSprayRetryPolicy() if options.CookieJar != nil { client.HTTPClient.Jar = options.CookieJar @@ -105,9 +101,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { client.HTTPClient.Jar = jar } - // initiate raw http client - rawClient := rawhttp.NewClient(rawhttp.DefaultOptions) - executer := &HTTPExecuter{ debug: options.Debug, jsonOutput: options.JSON, @@ -257,17 +250,6 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { return result } - pipeOptions := rawhttp.DefaultPipelineOptions - pipeOptions.Host = URL.Host - pipeOptions.MaxConnections = 1 - if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 { - pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections - } - if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 { - pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection - } - pipeclient := rawhttp.NewPipelineClient(pipeOptions) - // defaultMaxWorkers should be a sufficient value to keep queues always full maxWorkers := defaultMaxWorkers // in case the queue is bigger increase the workers diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index aa485b1cb..cbeff1d08 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -18,8 +18,8 @@ import ( "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/internal/collaborator" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spaolacci/murmur3" - "github.com/spf13/cast" ) const ( @@ -35,110 +35,110 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { functions := make(map[string]govaluate.ExpressionFunction) functions["len"] = func(args ...interface{}) (interface{}, error) { - length := len(cast.ToString(args[0])) + length := len(types.ToString(args[0])) return float64(length), nil } functions["toupper"] = func(args ...interface{}) (interface{}, error) { - return strings.ToUpper(cast.ToString(args[0])), nil + return strings.ToUpper(types.ToString(args[0])), nil } functions["tolower"] = func(args ...interface{}) (interface{}, error) { - return strings.ToLower(cast.ToString(args[0])), nil + return strings.ToLower(types.ToString(args[0])), nil } functions["replace"] = func(args ...interface{}) (interface{}, error) { - return strings.ReplaceAll(cast.ToString(args[0]), cast.ToString(args[1]), cast.ToString(args[2])), nil + return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil } functions["replace_regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(cast.ToString(args[1])) + compiled, err := regexp.Compile(types.ToString(args[1])) if err != nil { return nil, err } - return compiled.ReplaceAllString(cast.ToString(args[0]), cast.ToString(args[2])), nil + return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil } functions["trim"] = func(args ...interface{}) (interface{}, error) { - return strings.Trim(cast.ToString(args[0]), cast.ToString(args[2])), nil + return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil } functions["trimleft"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimLeft(cast.ToString(args[0]), cast.ToString(args[1])), nil + return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil } functions["trimright"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimRight(cast.ToString(args[0]), cast.ToString(args[1])), nil + return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil } functions["trimspace"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSpace(cast.ToString(args[0])), nil + return strings.TrimSpace(types.ToString(args[0])), nil } functions["trimprefix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimPrefix(cast.ToString(args[0]), cast.ToString(args[1])), nil + return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil } functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) { - return strings.TrimSuffix(cast.ToString(args[0]), cast.ToString(args[1])), nil + return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil } functions["reverse"] = func(args ...interface{}) (interface{}, error) { - return reverseString(cast.ToString(args[0])), nil + return reverseString(types.ToString(args[0])), nil } // encoding functions["base64"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0]))) + sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return sEnc, nil } // python encodes to base64 with lines of 76 bytes terminated by new line "\n" functions["base64_py"] = func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0]))) + sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return insertInto(sEnc, 76, '\n'), nil } functions["base64_decode"] = func(args ...interface{}) (interface{}, error) { - return base64.StdEncoding.DecodeString(cast.ToString(args[0])) + return base64.StdEncoding.DecodeString(types.ToString(args[0])) } functions["url_encode"] = func(args ...interface{}) (interface{}, error) { - return url.PathEscape(cast.ToString(args[0])), nil + return url.PathEscape(types.ToString(args[0])), nil } functions["url_decode"] = func(args ...interface{}) (interface{}, error) { - return url.PathUnescape(cast.ToString(args[0])) + return url.PathUnescape(types.ToString(args[0])) } functions["hex_encode"] = func(args ...interface{}) (interface{}, error) { - return hex.EncodeToString([]byte(cast.ToString(args[0]))), nil + return hex.EncodeToString([]byte(types.ToString(args[0]))), nil } functions["hex_decode"] = func(args ...interface{}) (interface{}, error) { - hx, _ := hex.DecodeString(cast.ToString(args[0])) + hx, _ := hex.DecodeString(types.ToString(args[0])) return string(hx), nil } functions["html_escape"] = func(args ...interface{}) (interface{}, error) { - return html.EscapeString(cast.ToString(args[0])), nil + return html.EscapeString(types.ToString(args[0])), nil } functions["html_unescape"] = func(args ...interface{}) (interface{}, error) { - return html.UnescapeString(cast.ToString(args[0])), nil + return html.UnescapeString(types.ToString(args[0])), nil } // hashing functions["md5"] = func(args ...interface{}) (interface{}, error) { - hash := md5.Sum([]byte(cast.ToString(args[0]))) + hash := md5.Sum([]byte(types.ToString(args[0]))) return hex.EncodeToString(hash[:]), nil } functions["sha256"] = func(args ...interface{}) (interface{}, error) { h := sha256.New() - _, err := h.Write([]byte(cast.ToString(args[0]))) + _, err := h.Write([]byte(types.ToString(args[0]))) if err != nil { return nil, err @@ -148,7 +148,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { functions["sha1"] = func(args ...interface{}) (interface{}, error) { h := sha1.New() - _, err := h.Write([]byte(cast.ToString(args[0]))) + _, err := h.Write([]byte(types.ToString(args[0]))) if err != nil { return nil, err @@ -157,20 +157,20 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } functions["mmh3"] = func(args ...interface{}) (interface{}, error) { - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(cast.ToString(args[0])), 0))), nil + return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil } // search functions["contains"] = func(args ...interface{}) (interface{}, error) { - return strings.Contains(cast.ToString(args[0]), cast.ToString(args[1])), nil + return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil } functions["regex"] = func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(cast.ToString(args[0])) + compiled, err := regexp.Compile(types.ToString(args[0])) if err != nil { return nil, err } - return compiled.MatchString(cast.ToString(args[1])), nil + return compiled.MatchString(types.ToString(args[1])), nil } // random generators @@ -178,10 +178,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { chars := letters + numbers bad := "" if len(args) >= 1 { - chars = cast.ToString(args[0]) + chars = types.ToString(args[0]) } if len(args) >= withCutSetArgsSize { - bad = cast.ToString(args[1]) + bad = types.ToString(args[1]) } chars = trimAll(chars, bad) return chars[rand.Intn(len(chars))], nil @@ -196,10 +196,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = cast.ToString(args[1]) + bad = types.ToString(args[1]) } if len(args) >= withBaseRandArgsSize { - base = cast.ToString(args[2]) + base = types.ToString(args[2]) } base = trimAll(base, bad) return randSeq(base, l), nil @@ -214,7 +214,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = cast.ToString(args[1]) + bad = types.ToString(args[1]) } chars = trimAll(chars, bad) return randSeq(chars, l), nil @@ -229,7 +229,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = cast.ToString(args[1]) + bad = types.ToString(args[1]) } chars = trimAll(chars, bad) return randSeq(chars, l), nil @@ -244,7 +244,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { l = args[0].(int) } if len(args) >= withCutSetArgsSize { - bad = cast.ToString(args[1]) + bad = types.ToString(args[1]) } chars = trimAll(chars, bad) return randSeq(chars, l), nil @@ -273,7 +273,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { // Collaborator functions["collab"] = func(args ...interface{}) (interface{}, error) { // check if collaborator contains a specific pattern - return collaborator.DefaultCollaborator.Has(cast.ToString(args[0])), nil + return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil } return functions } diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index ede5dedd4..2a26a6a44 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -25,14 +25,8 @@ func (e *Extractor) CompileExtractors() error { } // Setup the part of the request to match, if any. - if e.Part != "" { - e.part, ok = PartTypes[e.Part] - if !ok { - return fmt.Errorf("unknown matcher part specified: %s", e.Part) - } - } else { - e.part = BodyPart + if e.Part == "" { + e.Part = "body" } - return nil } diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index f6e2df400..0269aaa39 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,52 +1,21 @@ package extractors -import ( - "net/http" +import "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/miekg/dns" -) +// Extract extracts data from an output structure based on user options +func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} { + part, ok := data[e.Part] + if !ok { + return nil + } + partString := types.ToString(part) -// Extract extracts response from the parts of request using a regex -func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} { switch e.extractorType { case RegexExtractor: - if e.part == BodyPart { - return e.extractRegex(body) - } else if e.part == HeaderPart { - return e.extractRegex(headers) - } else { - matches := e.extractRegex(headers) - if len(matches) > 0 { - return matches - } - return e.extractRegex(body) - } + return e.extractRegex(partString) case KValExtractor: - if e.part == HeaderPart { - return e.extractKVal(resp) - } - - matches := e.extractKVal(resp) - - if len(matches) > 0 { - return matches - } - - return e.extractCookieKVal(resp) + return e.extractKVal(data) } - - return nil -} - -// ExtractDNS extracts response from dns message using a regex -// nolint:interfacer // dns.Msg is out of current scope -func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} { - switch e.extractorType { - case RegexExtractor: - return e.extractRegex(msg.String()) - case KValExtractor: - } - return nil } @@ -57,39 +26,34 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} { groupPlusOne := e.RegexGroup + 1 for _, regex := range e.regexCompiled { matches := regex.FindAllStringSubmatch(corpus, -1) + for _, match := range matches { - if len(match) >= groupPlusOne { - results[match[e.RegexGroup]] = struct{}{} + if len(match) < groupPlusOne { + continue + } + matchString := match[e.RegexGroup] + + if _, ok := results[matchString]; !ok { + results[matchString] = struct{}{} } } } return results } -// extractKVal extracts text from http response -func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} { +// extractKVal extracts key value pairs from a data map +func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) for _, k := range e.KVal { - for _, v := range r.Header.Values(k) { - results[v] = struct{}{} + item, ok := data[k] + if !ok { + continue + } + itemString := types.ToString(item) + if _, ok := results[itemString]; !ok { + results[itemString] = struct{}{} } } - - return results -} - -// extractCookieKVal extracts text from cookies -func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} { - results := make(map[string]struct{}) - - for _, k := range e.KVal { - for _, cookie := range r.Cookies() { - if cookie.Name == k { - results[cookie.Value] = struct{}{} - } - } - } - return results } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 4f78d5042..f800e6a0e 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -25,8 +25,6 @@ type Extractor struct { // // By default, matching is performed in request body. Part string `yaml:"part,omitempty"` - // part is the part of the request to match - part Part // Internal defines if this is used internally Internal bool `yaml:"internal,omitempty"` } @@ -46,27 +44,3 @@ var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, } - -// Part is the part of the request to match -type Part int - -const ( - // BodyPart matches body of the response. - BodyPart Part = iota + 1 - // HeaderPart matches headers of the response. - HeaderPart - // AllPart matches both response body and headers of the response. - AllPart -) - -// PartTypes is an table for conversion of part type from string. -var PartTypes = map[string]Part{ - "body": BodyPart, - "header": HeaderPart, - "all": AllPart, -} - -// GetPart returns the part of the matcher -func (e *Extractor) GetPart() Part { - return e.part -} diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index abf862140..bf3106298 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -17,9 +17,9 @@ func (m *Matcher) CompileMatchers() error { if !ok { return fmt.Errorf("unknown matcher type specified: %s", m.Type) } - // By default, match on all if user hasn't provided any specific items + // By default, match on body if user hasn't provided any specific items if m.Part == "" { - m.Part = "all" + m.Part = "body" } // Compile the regexes diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index 53adaf9aa..1fed42e67 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -4,7 +4,7 @@ import ( "bytes" "errors" - "github.com/spf13/cast" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // formatScreen formats the output for showing on screen. @@ -52,7 +52,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { if ok { builder.WriteString(" [") - extractorResults := cast.ToStringSlice(extractedResults) + extractorResults := types.ToStringSlice(extractedResults) for i, item := range extractorResults { builder.WriteString(w.aurora.BrightCyan(item).String()) @@ -68,7 +68,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { if ok { builder.WriteString(" [") - metaResults := cast.ToStringMap(metaResults) + metaResults := types.ToStringMap(metaResults) var first = true for name, value := range metaResults { @@ -79,7 +79,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { builder.WriteString(w.aurora.BrightYellow(name).String()) builder.WriteRune('=') - builder.WriteString(w.aurora.BrightYellow(cast.ToString(value)).String()) + builder.WriteString(w.aurora.BrightYellow(types.ToString(value)).String()) } builder.WriteString("]") } diff --git a/v2/pkg/protocols/dns/matchers.go b/v2/pkg/protocols/dns/matchers.go index 1e32bfffa..bcddea569 100644 --- a/v2/pkg/protocols/dns/matchers.go +++ b/v2/pkg/protocols/dns/matchers.go @@ -35,7 +35,9 @@ func responseToDSLMap(msg *dns.Msg) map[string]interface{} { data["ns"] = buffer.String() buffer.Reset() - data["raw"] = msg.String() + rawData := msg.String() + data["raw"] = rawData + data["body"] = rawData // Use rawdata as body for dns responses matching data["status_code"] = msg.Rcode return data } diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/matchers.go index 6dcfb62ab..38d6fb392 100644 --- a/v2/pkg/protocols/http/matchers.go +++ b/v2/pkg/protocols/http/matchers.go @@ -9,7 +9,7 @@ import ( // responseToDSLMap converts a HTTP response to a map for use in DSL matching func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+6+len(resp.Header)) + data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } @@ -18,14 +18,20 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D data["status_code"] = resp.StatusCode data["body"] = body + for _, cookie := range resp.Cookies() { + data[cookie.Name] = cookie.Value + } for k, v := range resp.Header { k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) data[k] = strings.Join(v, " ") } - data["headers"] = headers + data["header"] = headers + data["all_headers"] = headers if r, err := httputil.DumpResponse(resp, true); err == nil { - data["raw"] = string(r) + rawString := string(r) + data["raw"] = rawString + data["all"] = rawString } data["duration"] = duration.Seconds() return data diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 974e793bd..299fae458 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -279,24 +279,9 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { parsedURL.Host = hostname } } - return parsedURL.String() } -// CustomHeaders valid for all requests -type CustomHeaders []string - -// String returns just a label -func (c *CustomHeaders) String() string { - return "Custom Global Headers" -} - -// Set a new global header -func (c *CustomHeaders) Set(value string) error { - *c = append(*c, value) - return nil -} - // Next returns the next generator by URL func (r *BulkHTTPRequest) Next(reqURL string) bool { return r.gsfm.Next(reqURL) diff --git a/v2/pkg/types/interfaces.go b/v2/pkg/types/interfaces.go new file mode 100644 index 000000000..e29bdc6b2 --- /dev/null +++ b/v2/pkg/types/interfaces.go @@ -0,0 +1,89 @@ +// Taken from https://github.com/spf13/cast. + +package types + +import ( + "fmt" + "strconv" + "strings" +) + +// ToString converts an interface to string in a quick way +func ToString(data interface{}) string { + switch s := data.(type) { + case string: + return s + case bool: + return strconv.FormatBool(s) + case float64: + return strconv.FormatFloat(s, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(s), 'f', -1, 32) + case int: + return strconv.Itoa(s) + case int64: + return strconv.FormatInt(s, 10) + case int32: + return strconv.Itoa(int(s)) + case int16: + return strconv.FormatInt(int64(s), 10) + case int8: + return strconv.FormatInt(int64(s), 10) + case uint: + return strconv.FormatUint(uint64(s), 10) + case uint64: + return strconv.FormatUint(uint64(s), 10) + case uint32: + return strconv.FormatUint(uint64(s), 10) + case uint16: + return strconv.FormatUint(uint64(s), 10) + case uint8: + return strconv.FormatUint(uint64(s), 10) + case []byte: + return string(s) + case fmt.Stringer: + return s.String() + case error: + return s.Error() + default: + return fmt.Sprintf("%v", data) + } +} + +// ToStringSlice casts an interface to a []string type. +func ToStringSlice(i interface{}) []string { + var a []string + + switch v := i.(type) { + case []interface{}: + for _, u := range v { + a = append(a, ToString(u)) + } + return a + case []string: + return v + case string: + return strings.Fields(v) + case interface{}: + return []string{ToString(v)} + default: + return nil + } +} + +// ToStringMap casts an interface to a map[string]interface{} type. +func ToStringMap(i interface{}) map[string]interface{} { + var m = map[string]interface{}{} + + switch v := i.(type) { + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = val + } + return m + case map[string]interface{}: + return v + default: + return nil + } +} From 2b50d99c0ccd584d4930ae2144171484564d3b3f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 24 Dec 2020 12:56:28 +0530 Subject: [PATCH 18/92] Misc work on extractor and replacer --- v2/pkg/executer/executer_http.go | 55 -------- v2/pkg/operators/operators.go | 61 +++++++++ .../protocols/common/generators/generators.go | 15 +-- v2/pkg/protocols/common/replacer/replacer.go | 31 +++++ v2/pkg/protocols/dns/dns.go | 3 +- v2/pkg/requests/dns-request.go | 121 ------------------ 6 files changed, 97 insertions(+), 189 deletions(-) create mode 100644 v2/pkg/protocols/common/replacer/replacer.go delete mode 100644 v2/pkg/requests/dns-request.go diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 89d0959f1..18e86193b 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -549,61 +549,6 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, result.Unlock() } - matcherCondition := e.bulkHTTPRequest.GetMatchersCondition() - for _, matcher := range e.bulkHTTPRequest.Matchers { - // Check if the matcher matched - if !matcher.Match(resp, body, headers, duration, matchData) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - return nil - } - } else { - // If the matcher has matched, and its an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition { - result.Lock() - result.Matches[matcher.Name] = nil - // probably redundant but ensures we snapshot current payload values when matchers are valid - result.Meta = request.Meta - result.GotResults = true - result.Unlock() - e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL) - } - } - } - - // All matchers have successfully completed so now start with the - // next task which is extraction of input from matchers. - var extractorResults, outputExtractorResults []string - - for _, extractor := range e.bulkHTTPRequest.Extractors { - for match := range extractor.Extract(resp, body, headers) { - if _, ok := dynamicvalues[extractor.Name]; !ok { - dynamicvalues[extractor.Name] = match - } - - extractorResults = append(extractorResults, match) - - if !extractor.Internal { - outputExtractorResults = append(outputExtractorResults, match) - } - } - // probably redundant but ensures we snapshot current payload values when extractors are valid - result.Lock() - result.Meta = request.Meta - result.Extractions[extractor.Name] = extractorResults - result.Unlock() - } - - // Write a final string of output if matcher type is - // AND or if we have extractors for the mechanism too. - if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL) - result.Lock() - result.GotResults = true - result.Unlock() - } - return nil } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index c60e153a1..1c983cff0 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -24,3 +24,64 @@ type Operators struct { func (r *Operators) GetMatchersCondition() matchers.ConditionType { return r.matchersCondition } + +// Result is a result structure created from operators running on data. +type Result struct { + // Matches is a map of matcher names that we matched + Matches map[string]struct{} + // Extracts contains all the data extracted from inputs + Extracts map[string][]string + // DynamicValues contains any dynamic values to be templated + DynamicValues map[string]string +} + +// Execute executes the operators on data and returns a result structure +func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { + matcherCondition := r.GetMatchersCondition() + + result := &Result{ + Matches: make(map[string]struct{}), + Extracts: make(map[string][]string), + DynamicValues: make(map[string]string), + } + for _, matcher := range r.Matchers { + // Check if the matcher matched + if !matcher.Match(data) { + // If the condition is AND we haven't matched, try next request. + if matcherCondition == matchers.ANDCondition { + return nil, false + } + } else { + // If the matcher has matched, and its an OR + // write the first output then move to next matcher. + if matcherCondition == matchers.ORCondition { + result.Matches[matcher.Name] = struct{}{} + } + } + } + + // All matchers have successfully completed so now start with the + // next task which is extraction of input from matchers. + var extractorResults, outputExtractorResults []string + for _, extractor := range r.Extractors { + for match := range extractor.Extract(data) { + extractorResults = append(extractorResults, match) + + if extractor.Internal { + if _, ok := result.DynamicValues[extractor.Name]; !ok { + result.DynamicValues[extractor.Name] = match + } + } else { + outputExtractorResults = append(outputExtractorResults, match) + } + } + result.Extracts[extractor.Name] = extractorResults + } + + // Write a final string of output if matcher type is + // AND or if we have extractors for the mechanism too. + if len(result.Extracts) > 0 || len(result.Matches) > 0 || matcherCondition == matchers.ANDCondition { + return result, true + } + return nil, false +} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index c07dbc324..27bfcab04 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -110,7 +110,7 @@ func (i *Iterator) sniperValue() map[string]interface{} { if !p.Next() { p.ResetPosition() } - values[p.Keyword()] = p.Value() + values[p.name] = p.Value() p.IncrementPosition() } return values @@ -124,7 +124,7 @@ func (i *Iterator) pitchforkValue() map[string]interface{} { if !p.Next() { p.ResetPosition() } - values[p.Keyword()] = p.Value() + values[p.name] = p.Value() p.IncrementPosition() } return values @@ -154,7 +154,7 @@ func (i *Iterator) clusterbombValue() map[string]interface{} { p.ResetPosition() signalNext = true } - values[p.Keyword()] = p.Value() + values[p.name] = p.Value() if first { p.IncrementPosition() first = false @@ -189,11 +189,6 @@ func (i *payloadIterator) Next() bool { return true } -// Position returns the position of reader in payload iterator -func (i *payloadIterator) Position() int { - return i.index -} - func (i *payloadIterator) ResetPosition() { i.index = 0 } @@ -210,7 +205,3 @@ func (i *payloadIterator) Value() string { func (i *payloadIterator) Total() int { return len(i.values) } - -func (i *payloadIterator) Keyword() string { - return i.name -} diff --git a/v2/pkg/protocols/common/replacer/replacer.go b/v2/pkg/protocols/common/replacer/replacer.go new file mode 100644 index 000000000..29f4aece5 --- /dev/null +++ b/v2/pkg/protocols/common/replacer/replacer.go @@ -0,0 +1,31 @@ +package replacer + +import ( + "fmt" + "strings" +) + +const ( + markerGeneral = "§" + markerParenthesisOpen = "{{" + markerParenthesisClose = "}}" +) + +// New creates a new replacer structure for values replacement on the fly. +func New(values map[string]interface{}) *strings.Replacer { + replacerItems := make([]string, 0, len(values)*4) + + for key, val := range values { + valueStr := fmt.Sprintf("%s", val) + + replacerItems = append(replacerItems, + fmt.Sprintf("%s%s%s", markerParenthesisOpen, key, markerParenthesisClose), + valueStr, + ) + replacerItems = append(replacerItems, + fmt.Sprintf("%s%s%s", markerGeneral, key, markerGeneral), + valueStr, + ) + } + return strings.NewReplacer(replacerItems...) +} diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 91cda7725..e02071217 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/miekg/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) // Request contains a DNS protocol request to be made from a template @@ -49,7 +50,7 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { var q dns.Question - replacer := newReplacer(map[string]interface{}{"FQDN": domain}) + replacer := replacer.New(map[string]interface{}{"FQDN": domain}) q.Name = dns.Fqdn(replacer.Replace(r.Name)) q.Qclass = classToInt(r.Class) diff --git a/v2/pkg/requests/dns-request.go b/v2/pkg/requests/dns-request.go deleted file mode 100644 index 863f7c638..000000000 --- a/v2/pkg/requests/dns-request.go +++ /dev/null @@ -1,121 +0,0 @@ -package requests - -import ( - "strings" - - "github.com/miekg/dns" - "github.com/projectdiscovery/nuclei/v2/pkg/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" -) - -// DNSRequest contains a request to be made from a template -type DNSRequest struct { - Recursion bool `yaml:"recursion"` - // Path contains the path/s for the request - Name string `yaml:"name"` - Type string `yaml:"type"` - Class string `yaml:"class"` - Retries int `yaml:"retries"` - // Raw contains a raw request - Raw string `yaml:"raw,omitempty"` - - // Matchers contains the detection mechanism for the request to identify - // whether the request was successful - Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` - // matchersCondition is internal condition for the matchers. - matchersCondition matchers.ConditionType - // MatchersCondition is the condition of the matchers - // whether to use AND or OR. Default is OR. - MatchersCondition string `yaml:"matchers-condition,omitempty"` - // Extractors contains the extraction mechanism for the request to identify - // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` -} - -// GetMatchersCondition returns the condition for the matcher -func (r *DNSRequest) GetMatchersCondition() matchers.ConditionType { - return r.matchersCondition -} - -// SetMatchersCondition sets the condition for the matcher -func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) { - r.matchersCondition = condition -} - -// Returns the total number of requests the YAML rule will perform -func (r *DNSRequest) GetRequestCount() int64 { - return 1 -} - -// MakeDNSRequest creates a *dns.Request from a request template -func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { - domain = dns.Fqdn(domain) - - // Build a request on the specified URL - req := new(dns.Msg) - req.Id = dns.Id() - req.RecursionDesired = r.Recursion - - var q dns.Question - - replacer := newReplacer(map[string]interface{}{"FQDN": domain}) - - q.Name = dns.Fqdn(replacer.Replace(r.Name)) - q.Qclass = toQClass(r.Class) - q.Qtype = toQType(r.Type) - - req.Question = append(req.Question, q) - - return req, nil -} - -func toQType(ttype string) (rtype uint16) { - ttype = strings.TrimSpace(strings.ToUpper(ttype)) - - switch ttype { - case "A": - rtype = dns.TypeA - case "NS": - rtype = dns.TypeNS - case "CNAME": - rtype = dns.TypeCNAME - case "SOA": - rtype = dns.TypeSOA - case "PTR": - rtype = dns.TypePTR - case "MX": - rtype = dns.TypeMX - case "TXT": - rtype = dns.TypeTXT - case "AAAA": - rtype = dns.TypeAAAA - default: - rtype = dns.TypeA - } - - return -} - -func toQClass(tclass string) (rclass uint16) { - tclass = strings.TrimSpace(strings.ToUpper(tclass)) - - switch tclass { - case "INET": - rclass = dns.ClassINET - case "CSNET": - rclass = dns.ClassCSNET - case "CHAOS": - rclass = dns.ClassCHAOS - case "HESIOD": - rclass = dns.ClassHESIOD - case "NONE": - rclass = dns.ClassNONE - case "ANY": - rclass = dns.ClassANY - default: - // Use INET by default. - rclass = dns.ClassINET - } - - return -} From 60789f4ba2238de35db0611b164c4e2ee6b2741f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 24 Dec 2020 20:47:41 +0530 Subject: [PATCH 19/92] More refactoring of nuclei packages --- v2/pkg/executer/executer_dns.go | 40 -------------- v2/pkg/operators/extractors/extract.go | 25 ++------- v2/pkg/operators/extractors/extractors.go | 5 ++ v2/pkg/operators/matchers/match.go | 55 +++++--------------- v2/pkg/operators/matchers/match_test.go | 10 ++-- v2/pkg/operators/matchers/matchers.go | 10 ++-- v2/pkg/operators/operators.go | 20 +++++-- v2/pkg/protocols/http/matchers.go | 63 ++++++++++++++++++++++- v2/pkg/protocols/protocols.go | 6 +++ 9 files changed, 116 insertions(+), 118 deletions(-) diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 86d5b4e0d..2b7bd8ca7 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -8,7 +8,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) @@ -95,45 +94,6 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result { fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } - matcherCondition := e.dnsRequest.GetMatchersCondition() - - for _, matcher := range e.dnsRequest.Matchers { - // Check if the matcher matched - if !matcher.MatchDNS(resp) { - // If the condition is AND we haven't matched, return. - if matcherCondition == matchers.ANDCondition { - return result - } - } else { - // If the matcher has matched, and its an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 { - e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil) - result.GotResults = true - } - } - } - - // All matchers have successfully completed so now start with the - // next task which is extraction of input from matchers. - var extractorResults []string - - for _, extractor := range e.dnsRequest.Extractors { - for match := range extractor.ExtractDNS(resp) { - if !extractor.Internal { - extractorResults = append(extractorResults, match) - } - } - } - - // Write a final string of output if matcher type is - // AND or if we have extractors for the mechanism too. - if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults) - - result.GotResults = true - } - return result } diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 0269aaa39..31c9aa6f2 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -2,25 +2,8 @@ package extractors import "github.com/projectdiscovery/nuclei/v2/pkg/types" -// Extract extracts data from an output structure based on user options -func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} { - part, ok := data[e.Part] - if !ok { - return nil - } - partString := types.ToString(part) - - switch e.extractorType { - case RegexExtractor: - return e.extractRegex(partString) - case KValExtractor: - return e.extractKVal(data) - } - return nil -} - -// extractRegex extracts text from a corpus and returns it -func (e *Extractor) extractRegex(corpus string) map[string]struct{} { +// ExtractRegex extracts text from a corpus and returns it +func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { results := make(map[string]struct{}) groupPlusOne := e.RegexGroup + 1 @@ -41,8 +24,8 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} { return results } -// extractKVal extracts key value pairs from a data map -func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} { +// ExtractKval extracts key value pairs from a data map +func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) for _, k := range e.KVal { diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index f800e6a0e..f593a1747 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -44,3 +44,8 @@ var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, } + +// GetType returns the type of the matcher +func (e *Extractor) GetType() ExtractorType { + return e.extractorType +} diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 19ba7d29d..2b31386c4 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -5,37 +5,8 @@ import ( "strings" ) -// Match matches a generic data response again a given matcher -func (m *Matcher) Match(data map[string]interface{}) bool { - part, ok := data[m.Part] - if !ok { - return false - } - partString := part.(string) - - switch m.matcherType { - case StatusMatcher: - statusCode, ok := data["status_code"] - if !ok { - return false - } - return m.isNegative(m.matchStatusCode(statusCode.(int))) - case SizeMatcher: - return m.isNegative(m.matchSizeCode(len(partString))) - case WordsMatcher: - return m.isNegative(m.matchWords(partString)) - case RegexMatcher: - return m.isNegative(m.matchRegex(partString)) - case BinaryMatcher: - return m.isNegative(m.matchBinary(partString)) - case DSLMatcher: - return m.isNegative(m.matchDSL(data)) - } - return false -} - -// matchStatusCode matches a status code check against an HTTP Response -func (m *Matcher) matchStatusCode(statusCode int) bool { +// MatchStatusCode matches a status code check against a corpus +func (m *Matcher) MatchStatusCode(statusCode int) bool { // Iterate over all the status codes accepted as valid // // Status codes don't support AND conditions. @@ -50,8 +21,8 @@ func (m *Matcher) matchStatusCode(statusCode int) bool { return false } -// matchStatusCode matches a size check against an HTTP Response -func (m *Matcher) matchSizeCode(length int) bool { +// MatchSize matches a size check against a corpus +func (m *Matcher) MatchSize(length int) bool { // Iterate over all the sizes accepted as valid // // Sizes codes don't support AND conditions. @@ -66,8 +37,8 @@ func (m *Matcher) matchSizeCode(length int) bool { return false } -// matchWords matches a word check against an HTTP Response/Headers. -func (m *Matcher) matchWords(corpus string) bool { +// MatchWords matches a word check against a corpus. +func (m *Matcher) MatchWords(corpus string) bool { // Iterate over all the words accepted as valid for i, word := range m.Words { // Continue if the word doesn't match @@ -94,8 +65,8 @@ func (m *Matcher) matchWords(corpus string) bool { return false } -// matchRegex matches a regex check against an HTTP Response/Headers. -func (m *Matcher) matchRegex(corpus string) bool { +// MatchRegex matches a regex check against a corpus +func (m *Matcher) MatchRegex(corpus string) bool { // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -122,8 +93,8 @@ func (m *Matcher) matchRegex(corpus string) bool { return false } -// matchWords matches a word check against an HTTP Response/Headers. -func (m *Matcher) matchBinary(corpus string) bool { +// MatchBinary matches a binary check against a corpus +func (m *Matcher) MatchBinary(corpus string) bool { // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match @@ -151,11 +122,11 @@ func (m *Matcher) matchBinary(corpus string) bool { return false } -// matchDSL matches on a generic map result -func (m *Matcher) matchDSL(mp map[string]interface{}) bool { +// MatchDSL matches on a generic map result +func (m *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid for i, expression := range m.dslCompiled { - result, err := expression.Evaluate(mp) + result, err := expression.Evaluate(data) if err != nil { continue } diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index 7d02ecf9a..2ab27c403 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -9,22 +9,22 @@ import ( func TestANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.matchWords("a b") + matched := m.MatchWords("a b") require.True(t, matched, "Could not match valid AND condition") - matched = m.matchWords("b") + matched = m.MatchWords("b") require.False(t, matched, "Could match invalid AND condition") } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.matchWords("a b") + matched := m.MatchWords("a b") require.True(t, matched, "Could not match valid OR condition") - matched = m.matchWords("b") + matched = m.MatchWords("b") require.True(t, matched, "Could not match valid OR condition") - matched = m.matchWords("c") + matched = m.MatchWords("c") require.False(t, matched, "Could match invalid OR condition") } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 930461beb..20297587c 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -88,11 +88,15 @@ var ConditionTypes = map[string]ConditionType{ "or": ORCondition, } -// isNegative reverts the results of the match if the matcher -// is of type negative. -func (m *Matcher) isNegative(data bool) bool { +// Result reverts the results of the match if the matcher is of type negative. +func (m *Matcher) Result(data bool) bool { if m.Negative { return !data } return data } + +// GetType returns the type of the matcher +func (m *Matcher) GetType() MatcherType { + return m.matcherType +} diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 1c983cff0..39cdce260 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -3,6 +3,7 @@ package operators import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" ) // Operators contains the operators that can be applied on protocols @@ -31,12 +32,20 @@ type Result struct { Matches map[string]struct{} // Extracts contains all the data extracted from inputs Extracts map[string][]string + // OutputExtracts is the list of extracts to be displayed on screen. + OutputExtracts []string // DynamicValues contains any dynamic values to be templated DynamicValues map[string]string } +// MatchFunc performs matching operation for a matcher on model and returns true or false. +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool + +// ExtractFunc performs extracting operation for a extractor on model and returns true or false. +type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { +func (r *Operators) Execute(data output.Event, match MatchFunc, extract ExtractFunc) (*Result, bool) { matcherCondition := r.GetMatchersCondition() result := &Result{ @@ -46,7 +55,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { } for _, matcher := range r.Matchers { // Check if the matcher matched - if !matcher.Match(data) { + if !match(data, matcher) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { return nil, false @@ -62,9 +71,10 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { // All matchers have successfully completed so now start with the // next task which is extraction of input from matchers. - var extractorResults, outputExtractorResults []string for _, extractor := range r.Extractors { - for match := range extractor.Extract(data) { + var extractorResults []string + + for match := range extract(data, extractor) { extractorResults = append(extractorResults, match) if extractor.Internal { @@ -72,7 +82,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) { result.DynamicValues[extractor.Name] = match } } else { - outputExtractorResults = append(outputExtractorResults, match) + result.OutputExtracts = append(result.OutputExtracts, match) } } result.Extracts[extractor.Name] = extractorResults diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/matchers.go index 38d6fb392..185f0c401 100644 --- a/v2/pkg/protocols/http/matchers.go +++ b/v2/pkg/protocols/http/matchers.go @@ -5,8 +5,69 @@ import ( "net/http/httputil" "strings" "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" ) +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + part, ok := data[matcher.Part] + if !ok { + return false + } + partString := part.(string) + + switch partString { + case "header": + partString = "all_headers" + case "all": + partString = "raw" + } + switch matcher.GetType() { + case matchers.StatusMatcher: + statusCode, ok := data["status_code"] + if !ok { + return false + } + return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(partString))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(partString)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(partString)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(partString)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "header": + partString = "all_headers" + case "all": + partString = "raw" + } + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(partString) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + // responseToDSLMap converts a HTTP response to a map for use in DSL matching func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies())) @@ -25,13 +86,11 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) data[k] = strings.Join(v, " ") } - data["header"] = headers data["all_headers"] = headers if r, err := httputil.DumpResponse(resp, true); err == nil { rawString := string(r) data["raw"] = rawString - data["all"] = rawString } data["duration"] = duration.Seconds() return data diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 09542784b..d6fbf5b3d 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,6 +1,8 @@ package protocols import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" @@ -12,6 +14,10 @@ type Executer interface { Compile(options ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int64 + // Match performs matching operation for a matcher on model and returns true or false. + Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Extract performs extracting operation for a extractor on model and returns true or false. + Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the protocol requests and returns an output event channel. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. From 4c4978cd1200ee70d25022c7981f08f7a69218de Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 25 Dec 2020 02:24:55 +0530 Subject: [PATCH 20/92] Modelling the data flow process and operations for executers --- v2/pkg/executer/executer_dns.go | 50 -------- v2/pkg/operators/operators.go | 22 ++++ v2/pkg/protocols/dns/dns.go | 37 +++++- v2/pkg/protocols/dns/execute.go | 88 ++++++++++++++ v2/pkg/protocols/dns/matchers.go | 43 ------- v2/pkg/protocols/dns/operators.go | 114 ++++++++++++++++++ .../http/{matchers.go => operators.go} | 25 +++- v2/pkg/protocols/protocols.go | 6 +- v2/pkg/types/types.go | 2 - 9 files changed, 280 insertions(+), 107 deletions(-) create mode 100644 v2/pkg/protocols/dns/execute.go delete mode 100644 v2/pkg/protocols/dns/matchers.go create mode 100644 v2/pkg/protocols/dns/operators.go rename v2/pkg/protocols/http/{matchers.go => operators.go} (82%) diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 2b7bd8ca7..f68201ac5 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -1,12 +1,6 @@ package executer import ( - "fmt" - "os" - "strings" - - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -49,50 +43,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { // ExecuteDNS executes the DNS request on a URL func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result { - result := &Result{} - if e.vhost { - parts := strings.Split(reqURL, ",") - reqURL = parts[0] - } - - // Parse the URL and return domain if URL. - var domain string - if isURL(reqURL) { - domain = extractDomain(reqURL) - } else { - domain = reqURL - } - - // Compile each request for the template based on the URL - compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain) - if err != nil { - e.traceLog.Request(e.template.ID, domain, "dns", err) - result.Error = errors.Wrap(err, "could not make dns request") - p.Drop(1) - return result - } - e.traceLog.Request(e.template.ID, domain, "dns", nil) - - if e.debug { - gologger.Infof("Dumped DNS request for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String()) - } - - // Send the request to the target servers - resp, err := e.dnsClient.Do(compiledRequest) - if err != nil { - result.Error = errors.Wrap(err, "could not send dns request") - p.Drop(1) - return result - } - p.Update() - - gologger.Verbosef("Sent for [%s] to %s\n", "dns-request", e.template.ID, reqURL) - - if e.debug { - gologger.Infof("Dumped DNS response for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", resp.String()) - } return result } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 39cdce260..ca50ec1ff 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -1,6 +1,7 @@ package operators import ( + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -21,6 +22,27 @@ type Operators struct { matchersCondition matchers.ConditionType } +// Compile compiles the operators as well as their corresponding matchers and extractors +func (r *Operators) Compile() error { + if r.MatchersCondition != "" { + r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition] + } else { + r.matchersCondition = matchers.ANDCondition + } + + for _, matcher := range r.Matchers { + if err := matcher.CompileMatchers(); err != nil { + return errors.Wrap(err, "could not compile matcher") + } + } + for _, extractor := range r.Extractors { + if err := extractor.CompileExtractors(); err != nil { + return errors.Wrap(err, "could not compile extractor") + } + } + return nil +} + // GetMatchersCondition returns the condition for the matchers func (r *Operators) GetMatchersCondition() matchers.ConditionType { return r.matchersCondition diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index e02071217..f634680cc 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -4,7 +4,12 @@ import ( "strings" "github.com/miekg/dns" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/clientpool" + "github.com/projectdiscovery/retryabledns" ) // Request contains a DNS protocol request to be made from a template @@ -22,15 +27,35 @@ type Request struct { // Raw contains a raw request Raw string `yaml:"raw,omitempty"` + // Operators for the current request go here. + *operators.Operators + // cache any variables that may be needed for operation. - class uint16 - questionType uint16 + class uint16 + question uint16 + dnsClient *retryabledns.Client + options *protocols.ExecuterOptions } // Compile compiles the protocol request for further execution. -func (r *Request) Compile() error { +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + // Create a dns client for the class + client, err := clientpool.Get(options.Options, &clientpool.Configuration{ + Retries: r.Retries, + }) + if err != nil { + return errors.Wrap(err, "could not get dns client") + } + r.dnsClient = client + + if r.Operators != nil { + if err := r.Operators.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + } r.class = classToInt(r.Class) - r.questionType = questionTypeToInt(r.Type) + r.options = options + r.question = questionTypeToInt(r.Type) return nil } @@ -53,8 +78,8 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { replacer := replacer.New(map[string]interface{}{"FQDN": domain}) q.Name = dns.Fqdn(replacer.Replace(r.Name)) - q.Qclass = classToInt(r.Class) - q.Qtype = questionTypeToInt(r.Type) + q.Qclass = r.class + q.Qtype = r.question req.Question = append(req.Question, q) return req, nil } diff --git a/v2/pkg/protocols/dns/execute.go b/v2/pkg/protocols/dns/execute.go new file mode 100644 index 000000000..ab1e37928 --- /dev/null +++ b/v2/pkg/protocols/dns/execute.go @@ -0,0 +1,88 @@ +package dns + +import ( + "fmt" + "net/url" + "os" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" +) + +// Execute executes the protocol requests and returns true or false if results were found. +func (r *Request) Execute(input string) (bool, error) { + +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string) ([]output.Event, error) { + // Parse the URL and return domain if URL. + var domain string + if isURL(input) { + domain = extractDomain(input) + } else { + domain = input + } + + // Compile each request for the template based on the URL + compiledRequest, err := r.Make(domain) + if err != nil { + r.options.Output.Request(r.options.TemplateID, domain, "dns", err) + // p.Drop(1) + return nil, errors.Wrap(err, "could not build request") + } + + if r.options.Options.Debug { + gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) + fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String()) + } + + // Send the request to the target servers + resp, err := r.dnsClient.Do(compiledRequest) + if err != nil { + r.options.Output.Request(r.options.TemplateID, domain, "dns", err) + //p.Drop(1) + return nil, errors.Wrap(err, "could not send dns request") + } + // p.Update() + r.options.Output.Request(r.options.TemplateID, domain, "dns", err) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) + + if r.options.Options.Debug { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) + fmt.Fprintf(os.Stderr, "%s\n", resp.String()) + } + ouputEvent := responseToDSLMap(resp) + + if r.Operators != nil { + result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + if !ok { + return nil, nil + } + + } + return +} + +// isURL tests a string to determine if it is a well-structured url or not. +func isURL(toTest string) bool { + _, err := url.ParseRequestURI(toTest) + if err != nil { + return false + } + u, err := url.Parse(toTest) + if err != nil || u.Scheme == "" || u.Host == "" { + return false + } + return true +} + +// extractDomain extracts the domain name of a URL +func extractDomain(theURL string) string { + u, err := url.Parse(theURL) + if err != nil { + return "" + } + return u.Hostname() +} diff --git a/v2/pkg/protocols/dns/matchers.go b/v2/pkg/protocols/dns/matchers.go deleted file mode 100644 index bcddea569..000000000 --- a/v2/pkg/protocols/dns/matchers.go +++ /dev/null @@ -1,43 +0,0 @@ -package dns - -import ( - "bytes" - - "github.com/miekg/dns" -) - -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func responseToDSLMap(msg *dns.Msg) map[string]interface{} { - data := make(map[string]interface{}, 6) - - buffer := &bytes.Buffer{} - for _, question := range msg.Question { - buffer.WriteString(question.String()) - } - data["question"] = buffer.String() - buffer.Reset() - - for _, extra := range msg.Extra { - buffer.WriteString(extra.String()) - } - data["extra"] = buffer.String() - buffer.Reset() - - for _, answer := range msg.Answer { - buffer.WriteString(answer.String()) - } - data["answer"] = buffer.String() - buffer.Reset() - - for _, ns := range msg.Ns { - buffer.WriteString(ns.String()) - } - data["ns"] = buffer.String() - buffer.Reset() - - rawData := msg.String() - data["raw"] = rawData - data["body"] = rawData // Use rawdata as body for dns responses matching - data["status_code"] = msg.Rcode - return data -} diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go new file mode 100644 index 000000000..200249a63 --- /dev/null +++ b/v2/pkg/protocols/dns/operators.go @@ -0,0 +1,114 @@ +package dns + +import ( + "bytes" + + "github.com/miekg/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + part, ok := data[matcher.Part] + if !ok { + return false + } + partString := part.(string) + + switch partString { + case "body", "all": + partString = "raw" + } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + + switch matcher.GetType() { + case matchers.StatusMatcher: + statusCode, ok := data["rcode"] + if !ok { + return false + } + return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(itemStr))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(itemStr)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(itemStr)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(itemStr)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "body", "all": + partString = "raw" + } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func responseToDSLMap(msg *dns.Msg) output.Event { + data := make(output.Event, 6) + + data["rcode"] = msg.Rcode + buffer := &bytes.Buffer{} + for _, question := range msg.Question { + buffer.WriteString(question.String()) + } + data["question"] = buffer.String() + buffer.Reset() + + for _, extra := range msg.Extra { + buffer.WriteString(extra.String()) + } + data["extra"] = buffer.String() + buffer.Reset() + + for _, answer := range msg.Answer { + buffer.WriteString(answer.String()) + } + data["answer"] = buffer.String() + buffer.Reset() + + for _, ns := range msg.Ns { + buffer.WriteString(ns.String()) + } + data["ns"] = buffer.String() + buffer.Reset() + + rawData := msg.String() + data["raw"] = rawData + return data +} diff --git a/v2/pkg/protocols/http/matchers.go b/v2/pkg/protocols/http/operators.go similarity index 82% rename from v2/pkg/protocols/http/matchers.go rename to v2/pkg/protocols/http/operators.go index 185f0c401..c68cf036f 100644 --- a/v2/pkg/protocols/http/matchers.go +++ b/v2/pkg/protocols/http/operators.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher @@ -24,6 +25,13 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case "all": partString = "raw" } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + switch matcher.GetType() { case matchers.StatusMatcher: statusCode, ok := data["status_code"] @@ -32,13 +40,13 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(partString))) + return matcher.Result(matcher.MatchSize(len(itemStr))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(partString)) + return matcher.Result(matcher.MatchWords(itemStr)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(partString)) + return matcher.Result(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(partString)) + return matcher.Result(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)) } @@ -59,9 +67,16 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext case "all": partString = "raw" } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(partString) + return extractor.ExtractRegex(itemStr) case extractors.KValExtractor: return extractor.ExtractKval(data) } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index d6fbf5b3d..3edc35c74 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -18,7 +18,7 @@ type Executer interface { Match(data map[string]interface{}, matcher *matchers.Matcher) bool // Extract performs extracting operation for a extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} - // Execute executes the protocol requests and returns an output event channel. + // Execute executes the protocol requests and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string) ([]output.Event, error) @@ -26,6 +26,10 @@ type Executer interface { // ExecuterOptions contains the configuration options for executer clients type ExecuterOptions struct { + // TemplateID is the ID of the template for the request + TemplateID string + // TemplateInfo contains information block of the template request + TemplateInfo map[string]string // Output is a writer interface for writing output events from executer. Output output.Writer // Options contains configuration options for the executer. diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index a046f73b9..2744cbf5a 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -6,8 +6,6 @@ import ( // Options contains the configuration options for nuclei scanner. type Options struct { - // Vhost marks the input specified as VHOST input - Vhost bool // RandomAgent generates random User-Agent RandomAgent bool // Metrics enables display of metrics via an http endpoint From 9d3958743ab87793959607575ef3b75ae75f8fca Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 25 Dec 2020 12:55:46 +0530 Subject: [PATCH 21/92] Grouping things, added more internal result types, restructuring --- v2/pkg/operators/operators.go | 3 +- v2/pkg/output/format_json.go | 10 +--- v2/pkg/output/format_screen.go | 51 +++++-------------- v2/pkg/output/output.go | 39 ++++++++++++-- v2/pkg/protocols/dns/dns.go | 4 +- .../clientpool.go | 2 +- v2/pkg/protocols/dns/execute.go | 18 +++++-- v2/pkg/protocols/dns/group.go | 13 +++++ v2/pkg/protocols/dns/operators.go | 4 +- .../clientpool.go | 2 +- v2/pkg/protocols/protocols.go | 2 +- 11 files changed, 86 insertions(+), 62 deletions(-) rename v2/pkg/protocols/dns/{clientpool => dnsclientpool}/clientpool.go (98%) create mode 100644 v2/pkg/protocols/dns/group.go rename v2/pkg/protocols/http/{clientpool => httpclientpool}/clientpool.go (99%) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index ca50ec1ff..960509045 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -4,7 +4,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" ) // Operators contains the operators that can be applied on protocols @@ -67,7 +66,7 @@ type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data output.Event, match MatchFunc, extract ExtractFunc) (*Result, bool) { +func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { matcherCondition := r.GetMatchersCondition() result := &Result{ diff --git a/v2/pkg/output/format_json.go b/v2/pkg/output/format_json.go index 93932c2a0..fa1c00372 100644 --- a/v2/pkg/output/format_json.go +++ b/v2/pkg/output/format_json.go @@ -2,13 +2,7 @@ package output import jsoniter "github.com/json-iterator/go" -var jsoniterCfg jsoniter.API - -func init() { - jsoniterCfg = jsoniter.Config{SortMapKeys: true}.Froze() -} - // formatJSON formats the output for json based formatting -func (w *StandardWriter) formatJSON(output Event) ([]byte, error) { - return jsoniterCfg.Marshal(output) +func (w *StandardWriter) formatJSON(output *WrappedEvent) ([]byte, error) { + return jsoniter.Marshal(output) } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index 1fed42e67..bfea4b506 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -2,61 +2,41 @@ package output import ( "bytes" - "errors" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // formatScreen formats the output for showing on screen. -func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { +func (w *StandardWriter) formatScreen(output *WrappedEvent) ([]byte, error) { builder := &bytes.Buffer{} if !w.noMetadata { - id, ok := output["id"] - if !ok { - return nil, errors.New("no template id found") - } builder.WriteRune('[') - builder.WriteString(w.aurora.BrightGreen(id.(string)).String()) + builder.WriteString(w.aurora.BrightGreen(output.TemplateID).String()) - matcherName, ok := output["matcher_name"] - if ok && matcherName != "" { + if output.MatcherName != "" { builder.WriteString(":") - builder.WriteString(w.aurora.BrightGreen(matcherName).Bold().String()) + builder.WriteString(w.aurora.BrightGreen(output.MatcherName).Bold().String()) } - outputType, ok := output["type"] - if !ok { - return nil, errors.New("no output type found") - } builder.WriteString("] [") - builder.WriteString(w.aurora.BrightBlue(outputType.(string)).String()) + builder.WriteString(w.aurora.BrightBlue(output.Type).String()) builder.WriteString("] ") - severity, ok := output["severity"] - if !ok { - return nil, errors.New("no output severity found") - } builder.WriteString("[") - builder.WriteString(w.severityMap[severity.(string)]) + builder.WriteString(w.severityMap[output.Info["severity"]]) builder.WriteString("] ") } - matched, ok := output["matched"] - if !ok { - return nil, errors.New("no matched url found") - } - builder.WriteString(matched.(string)) + builder.WriteString(output.Matched) // If any extractors, write the results - extractedResults, ok := output["extracted_results"] - if ok { + if len(output.ExtractedResults) > 0 { builder.WriteString(" [") - extractorResults := types.ToStringSlice(extractedResults) - for i, item := range extractorResults { + for i, item := range output.ExtractedResults { builder.WriteString(w.aurora.BrightCyan(item).String()) - if i != len(extractorResults)-1 { + if i != len(output.ExtractedResults)-1 { builder.WriteRune(',') } } @@ -64,15 +44,12 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) { } // Write meta if any - metaResults, ok := output["meta"] - if ok { + if len(output.Metadata) > 0 { builder.WriteString(" [") - metaResults := types.ToStringMap(metaResults) - - var first = true - for name, value := range metaResults { - if first { + var first bool = true + for name, value := range output.Metadata { + if !first { builder.WriteRune(',') } first = false diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 60de992c1..5c27a58b1 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -8,6 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" ) // Writer is an interface which writes output to somewhere for nuclei events. @@ -17,7 +18,7 @@ type Writer interface { // Colorizer returns the colorizer instance for writer Colorizer() aurora.Aurora // Write writes the event to file and/or screen. - Write(Event) error + Write(*WrappedEvent) error // Request writes a log the requests trace log Request(templateID, url, requestType string, err error) } @@ -41,8 +42,38 @@ const ( var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) -// Event is a single output structure from nuclei. -type Event map[string]interface{} +// InternalEvent is an internal output generation structure for nuclei. +type InternalEvent map[string]interface{} + +// InternalWrappedEvent is a wrapped event with operators result added to it. +type InternalWrappedEvent struct { + InternalEvent InternalEvent + OperatorsResult *operators.Result +} + +// WrappedEvent is a wrapped result event for a single nuclei output. +type WrappedEvent struct { + // TemplateID is the ID of the template for the result. + TemplateID string `json:"templateID"` + // Info contains information block of the template for the result. + Info map[string]string `json:"info"` + // MatcherName is the name of the matcher matched if any. + MatcherName string `json:"matcher_name,omitempty"` + // Type is the type of the result event. + Type string `json:"type"` + // Host is the host input on which match was found. + Host string `json:"host,omitempty"` + // Matched contains the matched input in its transformed form. + Matched string `json:"matched,omitempty"` + // ExtractedResults contains the extraction result from the inputs. + ExtractedResults []string `json:"extracted_results,omitempty"` + // Request is the optional dumped request for the match. + Request string `json:"request,omitempty"` + // Response is the optional dumped response for the match. + Response string `json:"response,omitempty"` + // Metadata contains any optional metadata for the event + Metadata map[string]interface{} `json:"meta,omitempty"` +} // NewStandardWriter creates a new output writer based on user configurations func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) { @@ -85,7 +116,7 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (* } // Write writes the event to file and/or screen. -func (w *StandardWriter) Write(event Event) error { +func (w *StandardWriter) Write(event *WrappedEvent) error { var data []byte var err error diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index f634680cc..304ead0eb 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -8,7 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/clientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/retryabledns" ) @@ -40,7 +40,7 @@ type Request struct { // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Create a dns client for the class - client, err := clientpool.Get(options.Options, &clientpool.Configuration{ + client, err := dnsclientpool.Get(options.Options, &dnsclientpool.Configuration{ Retries: r.Retries, }) if err != nil { diff --git a/v2/pkg/protocols/dns/clientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go similarity index 98% rename from v2/pkg/protocols/dns/clientpool/clientpool.go rename to v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 442dc8e5e..8b05f093e 100644 --- a/v2/pkg/protocols/dns/clientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -1,4 +1,4 @@ -package clientpool +package dnsclientpool import ( "strconv" diff --git a/v2/pkg/protocols/dns/execute.go b/v2/pkg/protocols/dns/execute.go index ab1e37928..1bbe786ae 100644 --- a/v2/pkg/protocols/dns/execute.go +++ b/v2/pkg/protocols/dns/execute.go @@ -12,11 +12,20 @@ import ( // Execute executes the protocol requests and returns true or false if results were found. func (r *Request) Execute(input string) (bool, error) { - + events, err := r.ExecuteWithResults(input) + if err != nil { + return false, err + } + // We've a match in the form of a event, so display. + for _, event := range events { + if event.Operators != nil { + r.options.Output.Write(event.Event) + } + } } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string) ([]output.Event, error) { +func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -55,14 +64,15 @@ func (r *Request) ExecuteWithResults(input string) ([]output.Event, error) { } ouputEvent := responseToDSLMap(resp) + event := []*output.WrappedEvent{&output.WrappedEvent{Event: ouputEvent}} if r.Operators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { return nil, nil } - + event[0].Operators = result } - return + return event, nil } // isURL tests a string to determine if it is a well-structured url or not. diff --git a/v2/pkg/protocols/dns/group.go b/v2/pkg/protocols/dns/group.go new file mode 100644 index 000000000..d686352d5 --- /dev/null +++ b/v2/pkg/protocols/dns/group.go @@ -0,0 +1,13 @@ +package dns + +import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + +// Group is a group of requests to be executed for a protocol. +type Group []protocols.Executer + +// Execute executes the group of protocol requests +func (g Group) Execute() { + for _, executer := range g { + executer.ExecuteWithResults() + } +} diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 200249a63..86581e4d4 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -79,8 +79,8 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func responseToDSLMap(msg *dns.Msg) output.Event { - data := make(output.Event, 6) +func responseToDSLMap(msg *dns.Msg) output.InternalEvent { + data := make(output.InternalEvent, 6) data["rcode"] = msg.Rcode buffer := &bytes.Buffer{} diff --git a/v2/pkg/protocols/http/clientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go similarity index 99% rename from v2/pkg/protocols/http/clientpool/clientpool.go rename to v2/pkg/protocols/http/httpclientpool/clientpool.go index 90bc2a89a..d0f94d223 100644 --- a/v2/pkg/protocols/http/clientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -1,4 +1,4 @@ -package clientpool +package httpclientpool import ( "context" diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 3edc35c74..7da84b787 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -21,7 +21,7 @@ type Executer interface { // Execute executes the protocol requests and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string) ([]output.Event, error) + ExecuteWithResults(input string) ([]output.InternalWrappedEvent, error) } // ExecuterOptions contains the configuration options for executer clients From 8bc59fafc44269f37b2751bd5c196a42c3472910 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 25 Dec 2020 20:33:52 +0530 Subject: [PATCH 22/92] Finalized first iteration of execution groups + protocols --- v2/internal/progress/progress.go | 16 ++-- v2/pkg/output/format_json.go | 2 +- v2/pkg/output/format_screen.go | 2 +- v2/pkg/output/output.go | 8 +- v2/pkg/protocols/dns/executer.go | 77 +++++++++++++++++++ v2/pkg/protocols/dns/group.go | 13 ---- v2/pkg/protocols/dns/operators.go | 52 ++++++++++--- .../protocols/dns/{execute.go => request.go} | 27 ++----- v2/pkg/protocols/protocols.go | 29 +++++-- 9 files changed, 162 insertions(+), 64 deletions(-) create mode 100644 v2/pkg/protocols/dns/executer.go delete mode 100644 v2/pkg/protocols/dns/group.go rename v2/pkg/protocols/dns/{execute.go => request.go} (78%) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index bffbd4c37..64cb7de01 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -53,7 +53,7 @@ func NewProgress(active, metrics bool, port int) (*Progress, error) { } go func() { if err := progress.server.ListenAndServe(); err != nil { - gologger.Warningf("Could not serve metrics: %s\n", err) + gologger.Warning().Msgf("Could not serve metrics: %s", err) } }() } @@ -71,7 +71,7 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { if p.active { if err := p.stats.Start(makePrintCallback(), p.tickDuration); err != nil { - gologger.Warningf("Couldn't start statistics: %s\n", err) + gologger.Warning().Msgf("Couldn't start statistics: %s", err) } } } @@ -81,15 +81,15 @@ func (p *Progress) AddToTotal(delta int64) { p.stats.IncrementCounter("total", int(delta)) } -// Update progress tracking information and increments the request counter by one unit. -func (p *Progress) Update() { +// IncrementRequests increments the requests counter by 1. +func (p *Progress) IncrementRequests() { p.stats.IncrementCounter("requests", 1) } -// Drop drops the specified number of requests from the progress bar total. -// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. -func (p *Progress) Drop(count int64) { +// DecrementRequests decrements the number of requests from total. +func (p *Progress) DecrementRequests(count int64) { // mimic dropping by incrementing the completed requests + p.stats.IncrementCounter("requests", int(count)) p.stats.IncrementCounter("errors", int(count)) } @@ -183,7 +183,7 @@ func fmtDuration(d time.Duration) string { func (p *Progress) Stop() { if p.active { if err := p.stats.Stop(); err != nil { - gologger.Warningf("Couldn't stop statistics: %s\n", err) + gologger.Warning().Msgf("Couldn't stop statistics: %s", err) } } if p.server != nil { diff --git a/v2/pkg/output/format_json.go b/v2/pkg/output/format_json.go index fa1c00372..0ca22961e 100644 --- a/v2/pkg/output/format_json.go +++ b/v2/pkg/output/format_json.go @@ -3,6 +3,6 @@ package output import jsoniter "github.com/json-iterator/go" // formatJSON formats the output for json based formatting -func (w *StandardWriter) formatJSON(output *WrappedEvent) ([]byte, error) { +func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) { return jsoniter.Marshal(output) } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index bfea4b506..dc2860554 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -7,7 +7,7 @@ import ( ) // formatScreen formats the output for showing on screen. -func (w *StandardWriter) formatScreen(output *WrappedEvent) ([]byte, error) { +func (w *StandardWriter) formatScreen(output *ResultEvent) ([]byte, error) { builder := &bytes.Buffer{} if !w.noMetadata { diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 5c27a58b1..81ea2c14e 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -18,7 +18,7 @@ type Writer interface { // Colorizer returns the colorizer instance for writer Colorizer() aurora.Aurora // Write writes the event to file and/or screen. - Write(*WrappedEvent) error + Write(*ResultEvent) error // Request writes a log the requests trace log Request(templateID, url, requestType string, err error) } @@ -51,8 +51,8 @@ type InternalWrappedEvent struct { OperatorsResult *operators.Result } -// WrappedEvent is a wrapped result event for a single nuclei output. -type WrappedEvent struct { +// ResultEvent is a wrapped result event for a single nuclei output. +type ResultEvent struct { // TemplateID is the ID of the template for the result. TemplateID string `json:"templateID"` // Info contains information block of the template for the result. @@ -116,7 +116,7 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (* } // Write writes the event to file and/or screen. -func (w *StandardWriter) Write(event *WrappedEvent) error { +func (w *StandardWriter) Write(event *ResultEvent) error { var data []byte var err error diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go new file mode 100644 index 000000000..15d819c83 --- /dev/null +++ b/v2/pkg/protocols/dns/executer.go @@ -0,0 +1,77 @@ +package dns + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Executer executes a group of requests for a protocol +type Executer struct { + requests []*Request + options *protocols.ExecuterOptions +} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { + return &Executer{requests: requests, options: options} +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + for _, request := range e.requests { + if err := request.Compile(e.options); err != nil { + return err + } + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int64 { + var count int64 + for _, request := range e.requests { + count += request.Requests() + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input) + if err != nil { + return false, err + } + + // If we have a result field, we should add a result to slice. + for _, event := range events { + if event.OperatorsResult != nil { + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) + } + } + } + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string) ([]*output.ResultEvent, error) { + var results []*output.ResultEvent + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input) + if err != nil { + return nil, err + } + for _, event := range events { + if event.OperatorsResult != nil { + results = append(results, req.makeResultEvent(event)...) + } + } + } + return results, nil +} diff --git a/v2/pkg/protocols/dns/group.go b/v2/pkg/protocols/dns/group.go deleted file mode 100644 index d686352d5..000000000 --- a/v2/pkg/protocols/dns/group.go +++ /dev/null @@ -1,13 +0,0 @@ -package dns - -import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - -// Group is a group of requests to be executed for a protocol. -type Group []protocols.Executer - -// Execute executes the group of protocol requests -func (g Group) Execute() { - for _, executer := range g { - executer.ExecuteWithResults() - } -} diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 86581e4d4..c0e91e599 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -78,37 +78,71 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func responseToDSLMap(msg *dns.Msg) output.InternalEvent { - data := make(output.InternalEvent, 6) +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) - data["rcode"] = msg.Rcode + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "dns", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Request = wrapped.InternalEvent["request"].(string) + data.Response = wrapped.InternalEvent["raw"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { + data := make(output.InternalEvent, 8) + + // Some data regarding the request metadata + data["host"] = host + data["matched"] = matched + data["request"] = req.String() + + data["rcode"] = resp.Rcode buffer := &bytes.Buffer{} - for _, question := range msg.Question { + for _, question := range resp.Question { buffer.WriteString(question.String()) } data["question"] = buffer.String() buffer.Reset() - for _, extra := range msg.Extra { + for _, extra := range resp.Extra { buffer.WriteString(extra.String()) } data["extra"] = buffer.String() buffer.Reset() - for _, answer := range msg.Answer { + for _, answer := range resp.Answer { buffer.WriteString(answer.String()) } data["answer"] = buffer.String() buffer.Reset() - for _, ns := range msg.Ns { + for _, ns := range resp.Ns { buffer.WriteString(ns.String()) } data["ns"] = buffer.String() buffer.Reset() - rawData := msg.String() + rawData := resp.String() data["raw"] = rawData return data } diff --git a/v2/pkg/protocols/dns/execute.go b/v2/pkg/protocols/dns/request.go similarity index 78% rename from v2/pkg/protocols/dns/execute.go rename to v2/pkg/protocols/dns/request.go index 1bbe786ae..ecbbdb207 100644 --- a/v2/pkg/protocols/dns/execute.go +++ b/v2/pkg/protocols/dns/request.go @@ -10,20 +10,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" ) -// Execute executes the protocol requests and returns true or false if results were found. -func (r *Request) Execute(input string) (bool, error) { - events, err := r.ExecuteWithResults(input) - if err != nil { - return false, err - } - // We've a match in the form of a event, so display. - for _, event := range events { - if event.Operators != nil { - r.options.Output.Write(event.Event) - } - } -} - // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { // Parse the URL and return domain if URL. @@ -38,7 +24,7 @@ func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEve compiledRequest, err := r.Make(domain) if err != nil { r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - // p.Drop(1) + r.options.Progress.DecrementRequests(1) return nil, errors.Wrap(err, "could not build request") } @@ -51,10 +37,11 @@ func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEve resp, err := r.dnsClient.Do(compiledRequest) if err != nil { r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - //p.Drop(1) + r.options.Progress.DecrementRequests(1) return nil, errors.Wrap(err, "could not send dns request") } - // p.Update() + r.options.Progress.IncrementRequests() + r.options.Output.Request(r.options.TemplateID, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) @@ -62,15 +49,15 @@ func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEve gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } - ouputEvent := responseToDSLMap(resp) + ouputEvent := responseToDSLMap(compiledRequest, resp, input, input) - event := []*output.WrappedEvent{&output.WrappedEvent{Event: ouputEvent}} + event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} if r.Operators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { return nil, nil } - event[0].Operators = result + event[0].OperatorsResult = result } return event, nil } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 7da84b787..1469b2968 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,6 +1,7 @@ package protocols import ( + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -8,20 +9,16 @@ import ( "go.uber.org/ratelimit" ) -// Executer is an interface implemented any protocol based request generator. +// Executer is an interface implemented any protocol based request executer. type Executer interface { - // Compile compiles the request generators preparing any requests possible. + // Compile compiles the execution generators preparing any requests possible. Compile(options ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int64 - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool - // Extract performs extracting operation for a extractor on model and returns true or false. - Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} - // Execute executes the protocol requests and returns true or false if results were found. + // Execute executes the protocol group and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string) ([]output.InternalWrappedEvent, error) + ExecuteWithResults(input string) ([]*output.ResultEvent, error) } // ExecuterOptions contains the configuration options for executer clients @@ -34,6 +31,22 @@ type ExecuterOptions struct { Output output.Writer // Options contains configuration options for the executer. Options *types.Options + // Progress is a progress client for scan reporting + Progress *progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. RateLimiter ratelimit.Limiter } + +// Request is an interface implemented any protocol based request generator. +type Request interface { + // Compile compiles the request generators preparing any requests possible. + Compile(options ExecuterOptions) error + // Requests returns the total number of requests the rule will perform + Requests() int64 + // Match performs matching operation for a matcher on model and returns true or false. + Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Extract performs extracting operation for a extractor on model and returns true or false. + Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. + ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) +} From 5a690ca6169521b124920aca9eabaabd97b7616d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 25 Dec 2020 20:39:09 +0530 Subject: [PATCH 23/92] More improvements, adding metadata for state between requests --- v2/pkg/protocols/dns/executer.go | 23 +++++++----- v2/pkg/protocols/dns/operators.go | 58 +++++++++++++++---------------- v2/pkg/protocols/dns/request.go | 5 ++- v2/pkg/protocols/protocols.go | 6 ++-- 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go index 15d819c83..a51d1a9de 100644 --- a/v2/pkg/protocols/dns/executer.go +++ b/v2/pkg/protocols/dns/executer.go @@ -11,6 +11,8 @@ type Executer struct { options *protocols.ExecuterOptions } +var _ protocols.Executer = &Executer{} + // NewExecuter creates a new request executer for list of requests func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { return &Executer{requests: requests, options: options} @@ -40,18 +42,20 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool for _, req := range e.requests { - events, err := req.ExecuteWithResults(input) + events, err := req.ExecuteWithResults(input, nil) if err != nil { return false, err } // If we have a result field, we should add a result to slice. for _, event := range events { - if event.OperatorsResult != nil { - for _, result := range req.makeResultEvent(event) { - results = true - e.options.Output.Write(result) - } + if event.OperatorsResult == nil { + continue + } + + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) } } } @@ -63,14 +67,15 @@ func (e *Executer) ExecuteWithResults(input string) ([]*output.ResultEvent, erro var results []*output.ResultEvent for _, req := range e.requests { - events, err := req.ExecuteWithResults(input) + events, err := req.ExecuteWithResults(input, nil) if err != nil { return nil, err } for _, event := range events { - if event.OperatorsResult != nil { - results = append(results, req.makeResultEvent(event)...) + if event.OperatorsResult == nil { + continue } + results = append(results, req.makeResultEvent(event)...) } } return results, nil diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index c0e91e599..af044aeb5 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -78,35 +78,6 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) - - data := output.ResultEvent{ - TemplateID: r.options.TemplateID, - Info: r.options.TemplateInfo, - Type: "dns", - Host: wrapped.InternalEvent["host"].(string), - Matched: wrapped.InternalEvent["matched"].(string), - ExtractedResults: wrapped.OperatorsResult.OutputExtracts, - } - if r.options.Options.JSONRequests { - data.Request = wrapped.InternalEvent["request"].(string) - data.Response = wrapped.InternalEvent["raw"].(string) - } - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data.MatcherName = k - results = append(results, &data) - } - } else { - results = append(results, &data) - } - return results -} - // responseToDSLMap converts a DNS response to a map for use in DSL matching func responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 8) @@ -146,3 +117,32 @@ func responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalE data["raw"] = rawData return data } + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) + + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "dns", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Request = wrapped.InternalEvent["request"].(string) + data.Response = wrapped.InternalEvent["raw"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ecbbdb207..d12e2d457 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -8,10 +8,13 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) +var _ protocols.Request = &Request{} + // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { // Parse the URL and return domain if URL. var domain string if isURL(input) { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 1469b2968..138109ec3 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -12,7 +12,7 @@ import ( // Executer is an interface implemented any protocol based request executer. type Executer interface { // Compile compiles the execution generators preparing any requests possible. - Compile(options ExecuterOptions) error + Compile() error // Requests returns the total number of requests the rule will perform Requests() int64 // Execute executes the protocol group and returns true or false if results were found. @@ -40,7 +40,7 @@ type ExecuterOptions struct { // Request is an interface implemented any protocol based request generator. type Request interface { // Compile compiles the request generators preparing any requests possible. - Compile(options ExecuterOptions) error + Compile(options *ExecuterOptions) error // Requests returns the total number of requests the rule will perform Requests() int64 // Match performs matching operation for a matcher on model and returns true or false. @@ -48,5 +48,5 @@ type Request interface { // Extract performs extracting operation for a extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) + ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) } From 5bd3438b4f08f8afb0a8486b03652776c96a4843 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 02:08:48 +0530 Subject: [PATCH 24/92] Rewrote workflows engine in a simpler manner --- v2/pkg/workflows/compile.go | 10 +- v2/pkg/workflows/generator.go | 95 +++++++------- v2/pkg/workflows/var.go | 233 ---------------------------------- v2/pkg/workflows/workflows.go | 24 ++-- 4 files changed, 62 insertions(+), 300 deletions(-) delete mode 100644 v2/pkg/workflows/var.go diff --git a/v2/pkg/workflows/compile.go b/v2/pkg/workflows/compile.go index ff7eed92c..ae40ba53e 100644 --- a/v2/pkg/workflows/compile.go +++ b/v2/pkg/workflows/compile.go @@ -22,14 +22,10 @@ func Parse(file string) (*Workflow, error) { return nil, err } - if len(workflow.Workflows) > 0 { - if err := workflow.generateLogicFromWorkflows(); err != nil { - return nil, errors.Wrap(err, "could not generate workflow") - } - } - if workflow.Logic == "" { - return nil, errors.New("no logic provided") + if len(workflow.Workflows) == 0 { + return nil, errors.New("no workflow defined") } + // Compile workflow here. workflow.path = file return workflow, nil } diff --git a/v2/pkg/workflows/generator.go b/v2/pkg/workflows/generator.go index 00bb24ca2..bb4e91898 100644 --- a/v2/pkg/workflows/generator.go +++ b/v2/pkg/workflows/generator.go @@ -1,78 +1,73 @@ package workflows -import ( - "errors" - "strings" +import "go.uber.org/atomic" - "github.com/segmentio/ksuid" -) +// RunWorkflow runs a workflow on an input and returns true or false +func (w *Workflow) RunWorkflow(input string) (bool, error) { + results := &atomic.Bool{} -// generateLogicFromWorkflows generates a workflow logic using the -// yaml based workflow declaration. -// -// The implementation is very basic and contains a simple yaml->tengo -// convertor that implements basic required features. -func (w *Workflow) generateLogicFromWorkflows() error { - w.Variables = make(map[string]string) - - workflowBuilder := &strings.Builder{} for _, template := range w.Workflows { - if err := w.generateTemplateFunc(template, workflowBuilder); err != nil { - return err + err := w.runWorkflowStep(template, input, results) + if err != nil { + return false, err } } - w.Logic = workflowBuilder.String() - return nil + return results.Load(), nil } -func (w *Workflow) generateTemplateFunc(template *WorkflowTemplate, workflowBuilder *strings.Builder) error { - builder := &strings.Builder{} - builder.WriteString("var_") - builder.WriteString(ksuid.New().String()) - ID := builder.String() - w.Variables[ID] = template.Template - - if len(template.Subtemplates) > 0 && len(template.Matchers) > 0 { - return errors.New("subtemplates and matchers cannot be present together") - } - workflowBuilder.WriteRune('\n') +// runWorkflowStep runs a workflow step for the workflow. It executes the workflow +// in a recursive manner running all subtemplates and matchers. +func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool) error { if len(template.Matchers) > 0 { - workflowBuilder.WriteString(ID) - workflowBuilder.WriteString("()\n") + output, err := template.executer.ExecuteWithResults(input) + if err != nil { + return err + } + if len(output) == 0 { + return nil + } for _, matcher := range template.Matchers { - if len(matcher.Subtemplates) == 0 { - return errors.New("no subtemplates present for matcher") - } - workflowBuilder.WriteString("\nif ") - workflowBuilder.WriteString(ID) - workflowBuilder.WriteString("[\"") - workflowBuilder.WriteString(matcher.Name) - workflowBuilder.WriteString("\"] {") + for _, item := range output { + if item.OperatorsResult == nil { + continue + } - for _, subtemplate := range matcher.Subtemplates { - if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil { - return err + _, matchOK := item.OperatorsResult.Matches[matcher.Name] + _, extractOK := item.OperatorsResult.Extracts[matcher.Name] + if !matchOK && !extractOK { + continue + } + + for _, subtemplate := range matcher.Subtemplates { + if err := w.runWorkflowStep(subtemplate, input, results); err != nil { + return err + } } } - workflowBuilder.WriteString("\n}") } } if len(template.Subtemplates) > 0 { - workflowBuilder.WriteString("if ") - workflowBuilder.WriteString(ID) - workflowBuilder.WriteString("() {") + output, err := template.executer.ExecuteWithResults(input) + if err != nil { + return err + } + if len(output) == 0 { + return nil + } for _, subtemplate := range template.Subtemplates { - if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil { + if err := w.runWorkflowStep(subtemplate, input, results); err != nil { return err } } - workflowBuilder.WriteString("\n}") } if len(template.Matchers) == 0 && len(template.Subtemplates) == 0 { - workflowBuilder.WriteString(ID) - workflowBuilder.WriteString("();") + matched, err := template.executer.Execute(input) + if err != nil { + return err + } + results.CAS(false, matched) } return nil } diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go deleted file mode 100644 index 5d304721e..000000000 --- a/v2/pkg/workflows/var.go +++ /dev/null @@ -1,233 +0,0 @@ -package workflows - -import ( - "sync" - - tengo "github.com/d5/tengo/v2" - "github.com/logrusorgru/aurora" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" - "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" - "github.com/projectdiscovery/nuclei/v2/pkg/executer" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" -) - -const two = 2 - -// NucleiVar within the scripting engine -type NucleiVar struct { - tengo.ObjectImpl - Templates []*Template - URL string - InternalVars map[string]interface{} - sync.RWMutex -} - -// Template contains HTTPOptions and DNSOptions for a single template -type Template struct { - HTTPOptions *executer.HTTPOptions - DNSOptions *executer.DNSOptions - Progress *progress.Progress -} - -// TypeName of the variable -func (n *NucleiVar) TypeName() string { - return "nuclei-var" -} - -// CanCall can be called from within the scripting engine -func (n *NucleiVar) CanCall() bool { - return true -} - -// Call logic - args[0]=headers, args[1]=payloads -func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { - n.InternalVars = make(map[string]interface{}) - headers := make(map[string]string) - externalVars := make(map[string]interface{}) - - // if external variables are specified and matches the template ones, these gets overwritten - if len(args) >= 1 { - headers = iterableToMapString(args[0]) - } - - // if external variables are specified and matches the template ones, these gets overwritten - if len(args) >= two { - externalVars = iterableToMap(args[1]) - } - - var gotResult atomicboolean.AtomBool - - for _, template := range n.Templates { - p := template.Progress - - if template.HTTPOptions != nil { - p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount()) - - for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP { - // apply externally supplied headers if any - request.Headers = generators.MergeMapsWithStrings(request.Headers, headers) - // apply externally supplied payloads if any - request.Payloads = generators.MergeMaps(request.Payloads, externalVars) - - template.HTTPOptions.BulkHTTPRequest = request - - if template.HTTPOptions.Colorizer == nil { - template.HTTPOptions.Colorizer = colorizer.NewNucleiColorizer(aurora.NewAurora(true)) - } - - httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions) - - if err != nil { - p.Drop(request.GetRequestCount()) - gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) - - continue - } - - result := httpExecuter.ExecuteHTTP(p, n.URL) - - if result.Error != nil { - gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) - continue - } - - if result.GotResults { - gotResult.Or(result.GotResults) - n.addResults(result) - } - } - } - - if template.DNSOptions != nil { - p.AddToTotal(template.DNSOptions.Template.GetDNSRequestCount()) - - for _, request := range template.DNSOptions.Template.RequestsDNS { - template.DNSOptions.DNSRequest = request - dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) - result := dnsExecuter.ExecuteDNS(p, n.URL) - - if result.Error != nil { - gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) - continue - } - - if result.GotResults { - gotResult.Or(result.GotResults) - n.addResults(result) - } - } - } - } - - if gotResult.Get() { - return tengo.TrueValue, nil - } - - return tengo.FalseValue, nil -} - -func (n *NucleiVar) IsFalsy() bool { - n.RLock() - defer n.RUnlock() - - return len(n.InternalVars) == 0 -} - -func (n *NucleiVar) addResults(r *executer.Result) { - n.RLock() - defer n.RUnlock() - - // add payload values as first, they will be accessible if not overwritter through - // payload_name (from template) => value - for k, v := range r.Meta { - n.InternalVars[k] = v - } - - for k := range r.Matches { - n.InternalVars[k] = true - } - - for k, v := range r.Extractions { - n.InternalVars[k] = v - } -} - -// IndexGet returns the value for the given key. -func (n *NucleiVar) IndexGet(index tengo.Object) (res tengo.Object, err error) { - strIdx, ok := tengo.ToString(index) - if !ok { - err = tengo.ErrInvalidIndexType - return - } - - r, ok := n.InternalVars[strIdx] - if !ok { - return tengo.UndefinedValue, nil - } - - switch rt := r.(type) { - case bool: - if rt { - res = tengo.TrueValue - } else { - res = tengo.FalseValue - } - case string: - res = &tengo.String{Value: rt} - case []string: - rr, ok := r.([]string) - if !ok { - break - } - - var resA []tengo.Object - - for _, rrr := range rr { - resA = append(resA, &tengo.String{Value: rrr}) - } - - res = &tengo.Array{Value: resA} - } - - return res, nil -} - -func iterableToMap(t tengo.Object) map[string]interface{} { - m := make(map[string]interface{}) - - if t.CanIterate() { - i := t.Iterate() - for i.Next() { - key, ok := tengo.ToString(i.Key()) - if !ok { - continue - } - - value := tengo.ToInterface(i.Value()) - m[key] = value - } - } - - return m -} - -func iterableToMapString(t tengo.Object) map[string]string { - m := make(map[string]string) - - if t.CanIterate() { - i := t.Iterate() - for i.Next() { - key, ok := tengo.ToString(i.Key()) - if !ok { - continue - } - - if value, ok := tengo.ToString(i.Value()); ok { - m[key] = value - } - } - } - return m -} diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index ba985a355..8edc313c0 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -1,32 +1,36 @@ package workflows +import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + // Workflow is a workflow to execute with chained requests, etc. type Workflow struct { // ID is the unique id for the template ID string `yaml:"id"` // Info contains information about the template Info map[string]string `yaml:"info"` - // CookieReuse makes all cookies shared by templates within the workflow - CookieReuse bool `yaml:"cookie-reuse,omitempty"` - // Variables contains the variables accessible to the pseudo-code - Variables map[string]string `yaml:"variables"` - // Logic contains the workflow pseudo-code - Logic string `yaml:"logic"` // Workflows is a yaml based workflow declaration code. Workflows []*WorkflowTemplate `yaml:"workflows"` - path string + + path string } // WorkflowTemplate is a template to be ran as part of a workflow type WorkflowTemplate struct { - Template string `yaml:"template"` - Matchers []*Matcher `yaml:"matchers"` + // Template is the template to run + Template string `yaml:"template"` + // Matchers perform name based matching to run subtemplates for a workflow. + Matchers []*Matcher `yaml:"matchers"` + // Subtemplates are ran if the template matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` + + executer protocols.Executer } // Matcher performs conditional matching on the workflow template results. type Matcher struct { - Name string `yaml:"name"` + // Name is the name of the item to match. + Name string `yaml:"name"` + // Subtemplates are ran if the name of matcher matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` } From 164a67353b5e4acb84a0435c6c03796d335fa741 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 02:09:16 +0530 Subject: [PATCH 25/92] MIsc --- v2/pkg/atomicboolean/bool.go | 42 ----- v2/pkg/executer/executer_dns.go | 51 ------ v2/pkg/executer/output_dns.go | 106 ------------- v2/pkg/output/output.go | 1 + v2/pkg/protocols/dns/executer.go | 16 +- v2/pkg/protocols/dns/operators.go | 7 +- v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/http/build_request.go | 211 +++++++++++++++++++++++++ v2/pkg/protocols/protocols.go | 2 +- v2/pkg/requests/bulk-http-request.go | 196 ----------------------- 10 files changed, 231 insertions(+), 403 deletions(-) delete mode 100644 v2/pkg/atomicboolean/bool.go delete mode 100644 v2/pkg/executer/executer_dns.go delete mode 100644 v2/pkg/executer/output_dns.go create mode 100644 v2/pkg/protocols/http/build_request.go diff --git a/v2/pkg/atomicboolean/bool.go b/v2/pkg/atomicboolean/bool.go deleted file mode 100644 index 829b78ec6..000000000 --- a/v2/pkg/atomicboolean/bool.go +++ /dev/null @@ -1,42 +0,0 @@ -package atomicboolean - -import ( - "sync" -) - -type AtomBool struct { - sync.RWMutex - flag bool -} - -func New() *AtomBool { - return &AtomBool{} -} - -func (b *AtomBool) Or(value bool) { - b.Lock() - defer b.Unlock() - - b.flag = b.flag || value -} - -func (b *AtomBool) And(value bool) { - b.Lock() - defer b.Unlock() - - b.flag = b.flag && value -} - -func (b *AtomBool) Set(value bool) { - b.Lock() - defer b.Unlock() - - b.flag = value -} - -func (b *AtomBool) Get() bool { - b.RLock() - defer b.RUnlock() //nolint - - return b.flag -} diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go deleted file mode 100644 index f68201ac5..000000000 --- a/v2/pkg/executer/executer_dns.go +++ /dev/null @@ -1,51 +0,0 @@ -package executer - -import ( - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" -) - -// DNSExecuter is a client for performing a DNS request -// for a template. -type DNSExecuter struct { - template *templates.Template -} - -// DNSOptions contains configuration options for the DNS executer. -type DNSOptions struct { - Template *templates.Template - DNSRequest *requests.DNSRequest -} - -// NewDNSExecuter creates a new DNS executer from a template -// and a DNS request query. -func NewDNSExecuter(options *DNSOptions) *DNSExecuter { - - executer := &DNSExecuter{ - debug: options.Debug, - noMeta: options.NoMeta, - jsonOutput: options.JSON, - traceLog: options.TraceLog, - jsonRequest: options.JSONRequests, - dnsClient: dnsClient, - vhost: options.VHost, - template: options.Template, - dnsRequest: options.DNSRequest, - writer: options.Writer, - coloredOutput: options.ColoredOutput, - colorizer: options.Colorizer, - decolorizer: options.Decolorizer, - ratelimiter: options.RateLimiter, - } - return executer -} - -// ExecuteDNS executes the DNS request on a URL -func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result { - - return result -} - -// Close closes the dns executer for a template. -func (e *DNSExecuter) Close() {} diff --git a/v2/pkg/executer/output_dns.go b/v2/pkg/executer/output_dns.go deleted file mode 100644 index b68543f23..000000000 --- a/v2/pkg/executer/output_dns.go +++ /dev/null @@ -1,106 +0,0 @@ -package executer - -import ( - "strings" - - "github.com/miekg/dns" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" -) - -// writeOutputDNS writes dns output to streams -// nolint:interfacer // dns.Msg is out of current scope -func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) { - if e.jsonOutput { - output := make(jsonOutput) - output["matched"] = domain - - if !e.noMeta { - output["template"] = e.template.ID - output["type"] = "dns" - output["host"] = domain - for k, v := range e.template.Info { - output[k] = v - } - if matcher != nil && len(matcher.Name) > 0 { - output["matcher_name"] = matcher.Name - } - if len(extractorResults) > 0 { - output["extracted_results"] = extractorResults - } - if e.jsonRequest { - output["request"] = req.String() - output["response"] = resp.String() - } - } - - data, err := jsoniter.Marshal(output) - if err != nil { - gologger.Warningf("Could not marshal json output: %s\n", err) - } - gologger.Silentf("%s", string(data)) - if e.writer != nil { - if err := e.writer.Write(data); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } - return - } - - builder := &strings.Builder{} - colorizer := e.colorizer - - if !e.noMeta { - builder.WriteRune('[') - builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String()) - - if matcher != nil && len(matcher.Name) > 0 { - builder.WriteString(":") - builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String()) - } - - builder.WriteString("] [") - builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String()) - builder.WriteString("] ") - - if e.template.Info["severity"] != "" { - builder.WriteString("[") - builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"])) - builder.WriteString("] ") - } - } - builder.WriteString(domain) - - // If any extractors, write the results - if len(extractorResults) > 0 && !e.noMeta { - builder.WriteString(" [") - - for i, result := range extractorResults { - builder.WriteString(colorizer.Colorizer.BrightCyan(result).String()) - - if i != len(extractorResults)-1 { - builder.WriteRune(',') - } - } - builder.WriteString("]") - } - builder.WriteRune('\n') - - // Write output to screen as well as any output file - message := builder.String() - gologger.Silentf("%s", message) - - if e.writer != nil { - if e.coloredOutput { - message = e.decolorizer.ReplaceAllString(message, "") - } - - if err := e.writer.WriteString(message); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } -} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 81ea2c14e..25085887e 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -48,6 +48,7 @@ type InternalEvent map[string]interface{} // InternalWrappedEvent is a wrapped event with operators result added to it. type InternalWrappedEvent struct { InternalEvent InternalEvent + Results []*ResultEvent OperatorsResult *operators.Result } diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go index a51d1a9de..ec919cfc5 100644 --- a/v2/pkg/protocols/dns/executer.go +++ b/v2/pkg/protocols/dns/executer.go @@ -21,7 +21,8 @@ func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Execu // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { for _, request := range e.requests { - if err := request.Compile(e.options); err != nil { + err := request.Compile(e.options) + if err != nil { return err } } @@ -46,6 +47,9 @@ func (e *Executer) Execute(input string) (bool, error) { if err != nil { return false, err } + if events == nil { + return false, nil + } // If we have a result field, we should add a result to slice. for _, event := range events { @@ -63,20 +67,24 @@ func (e *Executer) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string) ([]*output.ResultEvent, error) { - var results []*output.ResultEvent +func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + var results []*output.InternalWrappedEvent for _, req := range e.requests { events, err := req.ExecuteWithResults(input, nil) if err != nil { return nil, err } + if events == nil { + return nil, nil + } for _, event := range events { if event.OperatorsResult == nil { continue } - results = append(results, req.makeResultEvent(event)...) + event.Results = req.makeResultEvent(event) } + results = append(results, events...) } return results, nil } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index af044aeb5..5ea850a9b 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -79,13 +79,16 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { +func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { data := make(output.InternalEvent, 8) // Some data regarding the request metadata data["host"] = host data["matched"] = matched - data["request"] = req.String() + + if r.options.Options.JSONRequests { + data["request"] = req.String() + } data["rcode"] = resp.Rcode buffer := &bytes.Buffer{} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index d12e2d457..4e25fcf65 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -52,7 +52,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } - ouputEvent := responseToDSLMap(compiledRequest, resp, input, input) + ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} if r.Operators != nil { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go new file mode 100644 index 000000000..eb81706a8 --- /dev/null +++ b/v2/pkg/protocols/http/build_request.go @@ -0,0 +1,211 @@ +package http + +import ( + "context" + "io" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/pkg/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser" +) + +// MakeHTTPRequest makes the HTTP request +func (r *Request) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) { + ctx := context.Background() + + parsed, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + + hostname := parsed.Host + + values := generators.MergeMaps(dynamicValues, map[string]interface{}{ + "BaseURL": baseURLWithTemplatePrefs(data, parsed), + "Hostname": hostname, + }) + + // if data contains \n it's a raw request + if strings.Contains(data, "\n") { + return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values) + } + return r.makeHTTPRequestFromModel(ctx, data, values) +} + +// MakeHTTPRequestFromModel creates a *http.Request from a request template +func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) { + replacer := newReplacer(values) + URL := replacer.Replace(data) + + // Build a request on the specified URL + req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil) + if err != nil { + return nil, err + } + + request, err := r.fillRequest(req, values) + if err != nil { + return nil, err + } + return &HTTPRequest{Request: request}, nil +} + +// InitGenerator initializes the generator +func (r *Request) InitGenerator() { + r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw) +} + +// CreateGenerator creates the generator +func (r *Request) CreateGenerator(reqURL string) { + r.gsfm.Add(reqURL) +} + +// HasGenerator check if an URL has a generator +func (r *Request) HasGenerator(reqURL string) bool { + return r.gsfm.Has(reqURL) +} + +// ReadOne reads and return a generator by URL +func (r *Request) ReadOne(reqURL string) { + r.gsfm.ReadOne(reqURL) +} + +// makeHTTPRequestFromRaw creates a *http.Request from a raw request +func (r *Request) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) { + // Add trailing line + data += "\n" + + if len(r.Payloads) > 0 { + r.gsfm.InitOrSkip(baseURL) + r.ReadOne(baseURL) + + payloads, err := r.GetPayloadsValues(baseURL) + if err != nil { + return nil, err + } + + return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) + } + + // otherwise continue with normal flow + return r.handleRawWithPaylods(ctx, data, baseURL, values, nil) +} + +func (r *Request) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) { + baseValues := generators.CopyMap(values) + finValues := generators.MergeMaps(baseValues, genValues) + + replacer := newReplacer(finValues) + + // Replace the dynamic variables in the URL if any + raw = replacer.Replace(raw) + + dynamicValues := make(map[string]interface{}) + // find all potentials tokens between {{}} + var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) + for _, match := range re.FindAllString(raw, -1) { + // check if the match contains a dynamic variable + expr := generators.TrimDelimiters(match) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) + + if err != nil { + return nil, err + } + + result, err := compiled.Evaluate(finValues) + if err != nil { + return nil, err + } + dynamicValues[expr] = result + } + + // replace dynamic values + dynamicReplacer := newReplacer(dynamicValues) + raw = dynamicReplacer.Replace(raw) + + rawRequest, err := r.parseRawRequest(raw, baseURL) + if err != nil { + return nil, err + } + + // rawhttp + if r.Unsafe { + unsafeReq := &HTTPRequest{ + RawRequest: rawRequest, + Meta: genValues, + AutomaticHostHeader: !r.DisableAutoHostname, + AutomaticContentLengthHeader: !r.DisableAutoContentLength, + Unsafe: true, + FollowRedirects: r.Redirects, + } + return unsafeReq, nil + } + + // retryablehttp + var body io.ReadCloser + body = ioutil.NopCloser(strings.NewReader(rawRequest.Data)) + if r.Race { + // More or less this ensures that all requests hit the endpoint at the same approximated time + // Todo: sync internally upon writing latest request byte + body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second) + } + + req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body) + if err != nil { + return nil, err + } + + // copy headers + for key, value := range rawRequest.Headers { + req.Header[key] = []string{value} + } + + request, err := r.fillRequest(req, values) + if err != nil { + return nil, err + } + + return &HTTPRequest{Request: request, Meta: genValues}, nil +} + +func (r *Request) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { + replacer := replacer.New(values) + // Set the header values requested + for header, value := range r.Headers { + req.Header[header] = []string{replacer.Replace(value)} + } + + // In case of multiple threads the underlying connection should remain open to allow reuse + if r.Threads <= 0 && req.Header.Get("Connection") == "" { + req.Close = true + } + + // Check if the user requested a request body + if r.Body != "" { + req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) + } + setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") + + // raw requests are left untouched + if len(r.Raw) > 0 { + return retryablehttp.FromRequest(req) + } + setHeader(req, "Accept", "*/*") + setHeader(req, "Accept-Language", "en") + + return retryablehttp.FromRequest(req) +} + +// setHeader sets some headers only if the header wasn't supplied by the user +func setHeader(req *http.Request, name, value string) { + if _, ok := req.Header[name]; !ok { + req.Header.Set(name, value) + } +} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 138109ec3..6aad81e73 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -18,7 +18,7 @@ type Executer interface { // Execute executes the protocol group and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string) ([]*output.ResultEvent, error) + ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) } // ExecuterOptions contains the configuration options for executer clients diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 299fae458..e801c0e7f 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -1,21 +1,15 @@ package requests import ( - "context" "fmt" - "io" - "io/ioutil" "net" "net/http" "net/url" "regexp" - "strings" - "time" "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/generators" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" ) @@ -52,196 +46,6 @@ func (r *BulkHTTPRequest) GetRequestCount() int64 { return int64(r.gsfm.Total()) } -// MakeHTTPRequest makes the HTTP request -func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) { - ctx := context.Background() - - parsed, err := url.Parse(baseURL) - if err != nil { - return nil, err - } - - hostname := parsed.Host - - values := generators.MergeMaps(dynamicValues, map[string]interface{}{ - "BaseURL": baseURLWithTemplatePrefs(data, parsed), - "Hostname": hostname, - }) - - // if data contains \n it's a raw request - if strings.Contains(data, "\n") { - return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values) - } - return r.makeHTTPRequestFromModel(ctx, data, values) -} - -// MakeHTTPRequestFromModel creates a *http.Request from a request template -func (r *BulkHTTPRequest) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) { - replacer := newReplacer(values) - URL := replacer.Replace(data) - - // Build a request on the specified URL - req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil) - if err != nil { - return nil, err - } - - request, err := r.fillRequest(req, values) - if err != nil { - return nil, err - } - return &HTTPRequest{Request: request}, nil -} - -// InitGenerator initializes the generator -func (r *BulkHTTPRequest) InitGenerator() { - r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw) -} - -// CreateGenerator creates the generator -func (r *BulkHTTPRequest) CreateGenerator(reqURL string) { - r.gsfm.Add(reqURL) -} - -// HasGenerator check if an URL has a generator -func (r *BulkHTTPRequest) HasGenerator(reqURL string) bool { - return r.gsfm.Has(reqURL) -} - -// ReadOne reads and return a generator by URL -func (r *BulkHTTPRequest) ReadOne(reqURL string) { - r.gsfm.ReadOne(reqURL) -} - -// makeHTTPRequestFromRaw creates a *http.Request from a raw request -func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) { - // Add trailing line - data += "\n" - - if len(r.Payloads) > 0 { - r.gsfm.InitOrSkip(baseURL) - r.ReadOne(baseURL) - - payloads, err := r.GetPayloadsValues(baseURL) - if err != nil { - return nil, err - } - - return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) - } - - // otherwise continue with normal flow - return r.handleRawWithPaylods(ctx, data, baseURL, values, nil) -} - -func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) { - baseValues := generators.CopyMap(values) - finValues := generators.MergeMaps(baseValues, genValues) - - replacer := newReplacer(finValues) - - // Replace the dynamic variables in the URL if any - raw = replacer.Replace(raw) - - dynamicValues := make(map[string]interface{}) - // find all potentials tokens between {{}} - var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) - for _, match := range re.FindAllString(raw, -1) { - // check if the match contains a dynamic variable - expr := generators.TrimDelimiters(match) - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) - - if err != nil { - return nil, err - } - - result, err := compiled.Evaluate(finValues) - if err != nil { - return nil, err - } - - dynamicValues[expr] = result - } - - // replace dynamic values - dynamicReplacer := newReplacer(dynamicValues) - raw = dynamicReplacer.Replace(raw) - - rawRequest, err := r.parseRawRequest(raw, baseURL) - if err != nil { - return nil, err - } - - // rawhttp - if r.Unsafe { - unsafeReq := &HTTPRequest{ - RawRequest: rawRequest, - Meta: genValues, - AutomaticHostHeader: !r.DisableAutoHostname, - AutomaticContentLengthHeader: !r.DisableAutoContentLength, - Unsafe: true, - FollowRedirects: r.Redirects, - } - return unsafeReq, nil - } - - // retryablehttp - var body io.ReadCloser - body = ioutil.NopCloser(strings.NewReader(rawRequest.Data)) - if r.Race { - // More or less this ensures that all requests hit the endpoint at the same approximated time - // Todo: sync internally upon writing latest request byte - body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second) - } - - req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body) - if err != nil { - return nil, err - } - - // copy headers - for key, value := range rawRequest.Headers { - req.Header[key] = []string{value} - } - - request, err := r.fillRequest(req, values) - if err != nil { - return nil, err - } - - return &HTTPRequest{Request: request, Meta: genValues}, nil -} - -func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { - replacer := newReplacer(values) - // Set the header values requested - for header, value := range r.Headers { - req.Header[header] = []string{replacer.Replace(value)} - } - - // In case of multiple threads the underlying connection should remain open to allow reuse - if r.Threads <= 0 && req.Header.Get("Connection") == "" { - req.Close = true - } - - // Check if the user requested a request body - if r.Body != "" { - req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) - } - - setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") - - // raw requests are left untouched - if len(r.Raw) > 0 { - return retryablehttp.FromRequest(req) - } - - setHeader(req, "Accept", "*/*") - setHeader(req, "Accept-Language", "en") - - return retryablehttp.FromRequest(req) -} - // HTTPRequest is the basic HTTP request type HTTPRequest struct { Request *retryablehttp.Request From 28485bd5aefa134952297b2dbe8e60b082fb4b60 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 13:20:56 +0530 Subject: [PATCH 26/92] Added tests + bug fixes in condition logic --- v2/pkg/workflows/{generator.go => execute.go} | 28 ++- v2/pkg/workflows/execute_test.go | 170 ++++++++++++++++++ 2 files changed, 182 insertions(+), 16 deletions(-) rename v2/pkg/workflows/{generator.go => execute.go} (86%) create mode 100644 v2/pkg/workflows/execute_test.go diff --git a/v2/pkg/workflows/generator.go b/v2/pkg/workflows/execute.go similarity index 86% rename from v2/pkg/workflows/generator.go rename to v2/pkg/workflows/execute.go index bb4e91898..b51f347ac 100644 --- a/v2/pkg/workflows/generator.go +++ b/v2/pkg/workflows/execute.go @@ -18,6 +18,16 @@ func (w *Workflow) RunWorkflow(input string) (bool, error) { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool) error { + var firstMatched bool + if len(template.Matchers) == 0 { + matched, err := template.executer.Execute(input) + if err != nil { + return err + } + firstMatched = matched + results.CAS(false, matched) + } + if len(template.Matchers) > 0 { output, err := template.executer.ExecuteWithResults(input) if err != nil { @@ -46,28 +56,14 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } } + return nil } - if len(template.Subtemplates) > 0 { - output, err := template.executer.ExecuteWithResults(input) - if err != nil { - return err - } - if len(output) == 0 { - return nil - } - + if len(template.Subtemplates) > 0 && firstMatched { for _, subtemplate := range template.Subtemplates { if err := w.runWorkflowStep(subtemplate, input, results); err != nil { return err } } } - if len(template.Matchers) == 0 && len(template.Subtemplates) == 0 { - matched, err := template.executer.Execute(input) - if err != nil { - return err - } - results.CAS(false, matched) - } return nil } diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go new file mode 100644 index 000000000..2822863c0 --- /dev/null +++ b/v2/pkg/workflows/execute_test.go @@ -0,0 +1,170 @@ +package workflows + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestWorkflowsSimple(t *testing.T) { + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true}}, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.True(t, matched, "could not get correct match value") +} + +func TestWorkflowsSimpleMultiple(t *testing.T) { + var firstInput, secondInput string + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }}}, + {executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}}, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.True(t, matched, "could not get correct match value") + + require.Equal(t, "https://test.com", firstInput, "could not get correct first input") + require.Equal(t, "https://test.com", secondInput, "could not get correct second input") +} + +func TestWorkflowsSubtemplates(t *testing.T) { + var firstInput, secondInput string + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }}, + Subtemplates: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}}, + }}, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.True(t, matched, "could not get correct match value") + + require.Equal(t, "https://test.com", firstInput, "could not get correct first input") + require.Equal(t, "https://test.com", secondInput, "could not get correct second input") +} + +func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { + var firstInput, secondInput string + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: false, executeHook: func(input string) { + firstInput = input + }}, + Subtemplates: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}}, + }}, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.False(t, matched, "could not get correct match value") + + require.Equal(t, "https://test.com", firstInput, "could not get correct first input") + require.Equal(t, "", secondInput, "could not get correct second input") +} + +func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { + var firstInput, secondInput string + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }, outputs: []*output.InternalWrappedEvent{ + {OperatorsResult: &operators.Result{ + Matches: map[string]struct{}{"tomcat": {}}, + Extracts: map[string][]string{}, + }}, + }}, + Matchers: []*Matcher{ + {Name: "tomcat", Subtemplates: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}}, + }}, + }, + }, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.True(t, matched, "could not get correct match value") + + require.Equal(t, "https://test.com", firstInput, "could not get correct first input") + require.Equal(t, "https://test.com", secondInput, "could not get correct second input") +} + +func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { + var firstInput, secondInput string + workflow := &Workflow{Workflows: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + firstInput = input + }, outputs: []*output.InternalWrappedEvent{ + {OperatorsResult: &operators.Result{ + Matches: map[string]struct{}{"tomcat": {}}, + Extracts: map[string][]string{}, + }}, + }}, + Matchers: []*Matcher{ + {Name: "apache", Subtemplates: []*WorkflowTemplate{ + {executer: &mockExecuter{result: true, executeHook: func(input string) { + secondInput = input + }}}, + }}, + }, + }, + }} + + matched, err := workflow.RunWorkflow("https://test.com") + require.Nil(t, err, "could not run workflow") + require.False(t, matched, "could not get correct match value") + + require.Equal(t, "https://test.com", firstInput, "could not get correct first input") + require.Equal(t, "", secondInput, "could not get correct second input") +} + +type mockExecuter struct { + result bool + executeHook func(input string) + outputs []*output.InternalWrappedEvent +} + +// Compile compiles the execution generators preparing any requests possible. +func (m *mockExecuter) Compile() error { + return nil +} + +// Requests returns the total number of requests the rule will perform +func (m *mockExecuter) Requests() int64 { + return 1 +} + +// Execute executes the protocol group and returns true or false if results were found. +func (m *mockExecuter) Execute(input string) (bool, error) { + if m.executeHook != nil { + m.executeHook(input) + } + return m.result, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (m *mockExecuter) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + if m.executeHook != nil { + m.executeHook(input) + } + return m.outputs, nil +} From 7b02ef9c01905484fd83bc70f6e4552bfe579fd9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 14:55:15 +0530 Subject: [PATCH 27/92] Starting to refactor http executer part --- v2/pkg/protocols/http/build_request.go | 110 +++- v2/pkg/protocols/http/http.go | 6 +- .../protocols/http/race/syncedreadcloser.go | 35 +- v2/pkg/protocols/http/raw/raw.go | 65 ++- v2/pkg/protocols/http/raw/raw_test.go | 4 +- v2/pkg/protocols/http/request.go | 500 ++++++++++++++++++ v2/pkg/requests/bulk-http-request.go | 94 ---- 7 files changed, 656 insertions(+), 158 deletions(-) create mode 100644 v2/pkg/protocols/http/request.go diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index eb81706a8..ba72280b7 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -2,8 +2,10 @@ package http import ( "context" + "fmt" "io" "io/ioutil" + "net" "net/http" "net/url" "regexp" @@ -11,11 +13,15 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/pkg/generators" + "github.com/projectdiscovery/nuclei/pkg/protcols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" - "github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/retryablehttp-go" ) +var urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) + // MakeHTTPRequest makes the HTTP request func (r *Request) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) { ctx := context.Background() @@ -98,19 +104,17 @@ func (r *Request) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data stri return r.handleRawWithPaylods(ctx, data, baseURL, values, nil) } -func (r *Request) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) { +func (r *Request) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) { baseValues := generators.CopyMap(values) finValues := generators.MergeMaps(baseValues, genValues) - replacer := newReplacer(finValues) - // Replace the dynamic variables in the URL if any - raw = replacer.Replace(raw) + rawRequest = replacer.New(finValues).Replace(rawRequest) dynamicValues := make(map[string]interface{}) // find all potentials tokens between {{}} var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) - for _, match := range re.FindAllString(raw, -1) { + for _, match := range re.FindAllString(rawRequest, -1) { // check if the match contains a dynamic variable expr := generators.TrimDelimiters(match) compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) @@ -126,11 +130,9 @@ func (r *Request) handleRawWithPaylods(ctx context.Context, raw, baseURL string, dynamicValues[expr] = result } - // replace dynamic values - dynamicReplacer := newReplacer(dynamicValues) - raw = dynamicReplacer.Replace(raw) - - rawRequest, err := r.parseRawRequest(raw, baseURL) + // Replacer dynamic values if any in raw request and parse it + rawRequest = replacer.New(dynamicValues).Replace(rawRequest) + rawRequestData, err := raw.Parse(rawRequest, baseURL, r.Unsafe) if err != nil { return nil, err } @@ -154,7 +156,7 @@ func (r *Request) handleRawWithPaylods(ctx context.Context, raw, baseURL string, if r.Race { // More or less this ensures that all requests hit the endpoint at the same approximated time // Todo: sync internally upon writing latest request byte - body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second) + body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second) } req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body) @@ -209,3 +211,85 @@ func setHeader(req *http.Request, name, value string) { req.Header.Set(name, value) } } + +// baseURLWithTemplatePrefs returns the url for BaseURL keeping +// the template port and path preference +func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { + // template port preference over input URL port + // template has port + hasPort := len(urlWithPortRegex.FindStringSubmatch(data)) > 0 + if hasPort { + // check if also the input contains port, in this case extracts the url + if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil { + parsedURL.Host = hostname + } + } + return parsedURL.String() +} + +// Next returns the next generator by URL +func (r *Request) Next(reqURL string) bool { + return r.gsfm.Next(reqURL) +} + +// Position returns the current generator's position by URL +func (r *Request) Position(reqURL string) int { + return r.gsfm.Position(reqURL) +} + +// Reset resets the generator by URL +func (r *Request) Reset(reqURL string) { + r.gsfm.Reset(reqURL) +} + +// Current returns the current generator by URL +func (r *Request) Current(reqURL string) string { + return r.gsfm.Current(reqURL) +} + +// Total is the total number of requests +func (r *Request) Total() int { + return r.gsfm.Total() +} + +// Increment increments the processed request +func (r *Request) Increment(reqURL string) { + r.gsfm.Increment(reqURL) +} + +// GetPayloadsValues for the specified URL +func (r *Request) GetPayloadsValues(reqURL string) (map[string]interface{}, error) { + payloadProcessedValues := make(map[string]interface{}) + payloadsFromTemplate := r.gsfm.Value(reqURL) + for k, v := range payloadsFromTemplate { + kexp := v.(string) + // if it doesn't containing markups, we just continue + if !hasMarker(kexp) { + payloadProcessedValues[k] = v + continue + } + // attempts to expand expressions + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions()) + if err != nil { + // it is a simple literal payload => proceed with literal value + payloadProcessedValues[k] = v + continue + } + // it is an expression - try to solve it + expValue, err := compiled.Evaluate(payloadsFromTemplate) + if err != nil { + // an error occurred => proceed with literal value + payloadProcessedValues[k] = v + continue + } + payloadProcessedValues[k] = fmt.Sprint(expValue) + } + var err error + if len(payloadProcessedValues) == 0 { + err = ErrNoPayload + } + return payloadProcessedValues, err +} + +// ErrNoPayload error to avoid the additional base null request +var ErrNoPayload = fmt.Errorf("no payload found") diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 3ab61eaf8..736972824 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,6 +1,9 @@ package http -import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" +) // Request contains a http request to be made from a template type Request struct { @@ -49,4 +52,5 @@ type Request struct { Race bool `yaml:"race"` attackType generators.Type + options *protocols.ExecuterOptions } diff --git a/v2/pkg/protocols/http/race/syncedreadcloser.go b/v2/pkg/protocols/http/race/syncedreadcloser.go index bdefaca34..c8f5de7ea 100644 --- a/v2/pkg/protocols/http/race/syncedreadcloser.go +++ b/v2/pkg/protocols/http/race/syncedreadcloser.go @@ -7,9 +7,9 @@ import ( "time" ) -// syncedReadCloser is compatible with io.ReadSeeker and performs +// SyncedReadCloser is compatible with io.ReadSeeker and performs // gate-based synced writes to enable race condition testing. -type syncedReadCloser struct { +type SyncedReadCloser struct { data []byte p int64 length int64 @@ -17,9 +17,10 @@ type syncedReadCloser struct { enableBlocking bool } -func newSyncedReadCloser(r io.ReadCloser) *syncedReadCloser { +// NewSyncedReadCloser creates a new SyncedReadCloser instance. +func NewSyncedReadCloser(r io.ReadCloser) *SyncedReadCloser { var ( - s syncedReadCloser + s SyncedReadCloser err error ) s.data, err = ioutil.ReadAll(r) @@ -33,27 +34,32 @@ func newSyncedReadCloser(r io.ReadCloser) *syncedReadCloser { return &s } -func newOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *syncedReadCloser { - s := newSyncedReadCloser(r) +// NewOpenGateWithTimeout creates a new open gate with a timeout +func NewOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *SyncedReadCloser { + s := NewSyncedReadCloser(r) s.OpenGateAfter(d) return s } -func (s *syncedReadCloser) SetOpenGate(status bool) { +// SetOpenGate sets the status of the blocking gate +func (s *SyncedReadCloser) SetOpenGate(status bool) { s.enableBlocking = status } -func (s *syncedReadCloser) OpenGate() { +// OpenGate opens the gate allowing all requests to be completed +func (s *SyncedReadCloser) OpenGate() { s.opengate <- struct{}{} } -func (s *syncedReadCloser) OpenGateAfter(d time.Duration) { +// OpenGateAfter schedules gate to be opened after a duration +func (s *SyncedReadCloser) OpenGateAfter(d time.Duration) { time.AfterFunc(d, func() { s.opengate <- struct{}{} }) } -func (s *syncedReadCloser) Seek(offset int64, whence int) (int64, error) { +// Seek implements seek method for io.ReadSeeker +func (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) { var err error switch whence { case io.SeekStart: @@ -74,7 +80,8 @@ func (s *syncedReadCloser) Seek(offset int64, whence int) (int64, error) { return s.p, err } -func (s *syncedReadCloser) Read(p []byte) (n int, err error) { +// Read implements read method for io.ReadSeeker +func (s *SyncedReadCloser) Read(p []byte) (n int, err error) { // If the data fits in the buffer blocks awaiting the sync instruction if s.p+int64(len(p)) >= s.length && s.enableBlocking { <-s.opengate @@ -87,10 +94,12 @@ func (s *syncedReadCloser) Read(p []byte) (n int, err error) { return n, err } -func (s *syncedReadCloser) Close() error { +// Close implements close method for io.ReadSeeker +func (s *SyncedReadCloser) Close() error { return nil } -func (s *syncedReadCloser) Len() int { +// Len returns the length of data in reader +func (s *SyncedReadCloser) Len() int { return int(s.length) } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 894c61f2c..b2d5eeab1 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -8,8 +8,9 @@ import ( "strings" ) -// Request defines a HTTP raw request structure +// Request defines a basic HTTP raw request type Request struct { + FullURL string Method string Path string Data string @@ -17,10 +18,9 @@ type Request struct { } // Parse parses the raw request as supplied by the user -func Parse(request string, unsafe bool) (*Request, error) { +func Parse(request, baseURL string, unsafe bool) (*Request, error) { reader := bufio.NewReader(strings.NewReader(request)) - - rawRequest := Request{ + rawRequest := &Request{ Headers: make(map[string]string), } @@ -30,7 +30,6 @@ func Parse(request string, unsafe bool) (*Request, error) { } parts := strings.Split(s, " ") - //nolint:gomnd // this is not a magic number if len(parts) < 3 { return nil, fmt.Errorf("malformed request supplied") @@ -79,40 +78,36 @@ func Parse(request string, unsafe bool) (*Request, error) { rawRequest.Path = parts[1] } + // If raw request doesn't have a Host header and/ path, + // this will be generated from the parsed baseURL + parsedURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + + var hostURL string + if rawRequest.Headers["Host"] == "" { + hostURL = parsedURL.Host + } else { + hostURL = rawRequest.Headers["Host"] + } + + if rawRequest.Path == "" { + rawRequest.Path = parsedURL.Path + } else if strings.HasPrefix(rawRequest.Path, "?") { + // requests generated from http.ReadRequest have incorrect RequestURI, so they + // cannot be used to perform another request directly, we need to generate a new one + // with the new target url + rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + } + + rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) + // Set the request body b, err := ioutil.ReadAll(reader) if err != nil { return nil, fmt.Errorf("could not read request body: %s", err) } rawRequest.Data = string(b) - return &rawRequest, nil -} - -// URL returns the full URL for a raw request based on provided metadata -func (r *Request) URL(BaseURL string) (string, error) { - parsed, err := url.Parse(BaseURL) - if err != nil { - return "", err - } - - var hostURL string - if r.Headers["Host"] == "" { - hostURL = parsed.Host - } else { - hostURL = r.Headers["Host"] - } - - if r.Path == "" { - r.Path = parsed.Path - } else if strings.HasPrefix(r.Path, "?") { - r.Path = fmt.Sprintf("%s%s", parsed.Path, r.Path) - } - - builder := &strings.Builder{} - builder.WriteString(parsed.Scheme) - builder.WriteString("://") - builder.WriteString(strings.TrimSpace(hostURL)) - builder.WriteString(r.Path) - URL := builder.String() - return URL, nil + return rawRequest, nil } diff --git a/v2/pkg/protocols/http/raw/raw_test.go b/v2/pkg/protocols/http/raw/raw_test.go index 0b00c0c43..ae439f3b2 100644 --- a/v2/pkg/protocols/http/raw/raw_test.go +++ b/v2/pkg/protocols/http/raw/raw_test.go @@ -12,7 +12,7 @@ Host: {{Hostname}} Authorization: Basic {{base64('username:password')}} User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0 Accept-Language: en-US,en;q=0.9 -Connection: close`, true) +Connection: close`, "https://test.com", true) require.Nil(t, err, "could not parse GET request") require.Equal(t, "GET", request.Method, "Could not parse GET method request correctly") require.Equal(t, "/manager/html", request.Path, "Could not parse request path correctly") @@ -21,7 +21,7 @@ Connection: close`, true) Host: {{Hostname}} Connection: close -username=admin&password=login`, true) +username=admin&password=login`, "https://test.com", true) require.Nil(t, err, "could not parse POST request") require.Equal(t, "POST", request.Method, "Could not parse POST method request correctly") require.Equal(t, "username=admin&password=login", request.Data, "Could not parse request data correctly") diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go new file mode 100644 index 000000000..f8639673d --- /dev/null +++ b/v2/pkg/protocols/http/request.go @@ -0,0 +1,500 @@ +package http + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/corpix/uarand" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/pkg/protcols/common/generators" + "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/requests" + "github.com/projectdiscovery/rawhttp" + "github.com/remeh/sizedwaitgroup" +) + +func (e *Request) ExecuteRaceRequest(reqURL string) *Result { + result := &Result{ + Matches: make(map[string]interface{}), + Extractions: make(map[string]interface{}), + } + + dynamicvalues := make(map[string]interface{}) + + // verify if the URL is already being processed + if e.HasGenerator(reqURL) { + return result + } + + e.CreateGenerator(reqURL) + + // Workers that keeps enqueuing new requests + maxWorkers := e.RaceNumberRequests + swg := sizedwaitgroup.New(maxWorkers) + for i := 0; i < e.RaceNumberRequests; i++ { + swg.Add() + // base request + result.Lock() + request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) + payloads, _ := e.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + } + go func(httpRequest *requests.HTTPRequest) { + defer swg.Done() + + // If the request was built correctly then execute it + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") + if err != nil { + result.Error = errors.Wrap(err, "could not handle http request") + } + }(request) + } + + swg.Wait() + + return result +} + +func (e *Request) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result { + result := &Result{ + Matches: make(map[string]interface{}), + Extractions: make(map[string]interface{}), + } + + dynamicvalues := make(map[string]interface{}) + + // verify if the URL is already being processed + if e.HasGenerator(reqURL) { + return result + } + + remaining := e.GetRequestCount() + e.CreateGenerator(reqURL) + + // Workers that keeps enqueuing new requests + maxWorkers := e.Threads + swg := sizedwaitgroup.New(maxWorkers) + for e.Next(reqURL) { + result.Lock() + request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) + payloads, _ := e.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + p.Drop(remaining) + } else { + swg.Add() + go func(httpRequest *requests.HTTPRequest) { + defer swg.Done() + + e.ratelimiter.Take() + + // If the request was built correctly then execute it + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") + if err != nil { + e.traceLog.Request(e.template.ID, reqURL, "http", err) + result.Error = errors.Wrap(err, "could not handle http request") + p.Drop(remaining) + } else { + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } + }(request) + } + p.Update() + e.Increment(reqURL) + } + swg.Wait() + + return result +} + +func (e *Request) ExecuteTurboHTTP(reqURL string) *Result { + result := &Result{ + Matches: make(map[string]interface{}), + Extractions: make(map[string]interface{}), + } + + dynamicvalues := make(map[string]interface{}) + + // verify if the URL is already being processed + if e.HasGenerator(reqURL) { + return result + } + + e.CreateGenerator(reqURL) + + // need to extract the target from the url + URL, err := url.Parse(reqURL) + if err != nil { + return result + } + + pipeOptions := rawhttp.DefaultPipelineOptions + pipeOptions.Host = URL.Host + pipeOptions.MaxConnections = 1 + if e.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = e.PipelineConcurrentConnections + } + if e.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = e.PipelineRequestsPerConnection + } + pipeclient := rawhttp.NewPipelineClient(pipeOptions) + + // defaultMaxWorkers should be a sufficient value to keep queues always full + maxWorkers := defaultMaxWorkers + // in case the queue is bigger increase the workers + if pipeOptions.MaxPendingRequests > maxWorkers { + maxWorkers = pipeOptions.MaxPendingRequests + } + swg := sizedwaitgroup.New(maxWorkers) + for e.Next(reqURL) { + result.Lock() + request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) + payloads, _ := e.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + } else { + swg.Add() + go func(httpRequest *requests.HTTPRequest) { + defer swg.Done() + + // HTTP pipelining ignores rate limit + // If the request was built correctly then execute it + request.Pipeline = true + request.PipelineClient = pipeclient + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") + if err != nil { + e.traceLog.Request(e.template.ID, reqURL, "http", err) + result.Error = errors.Wrap(err, "could not handle http request") + } else { + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } + request.PipelineClient = nil + }(request) + } + + e.Increment(reqURL) + } + swg.Wait() + return result +} + +// ExecuteHTTP executes the HTTP request on a URL +func (e *Request) ExecuteHTTP(p *progress.Progress, reqURL string) *Result { + // verify if pipeline was requested + if e.Pipeline { + return e.ExecuteTurboHTTP(reqURL) + } + + // verify if a basic race condition was requested + if e.Race && e.RaceNumberRequests > 0 { + return e.ExecuteRaceRequest(reqURL) + } + + // verify if parallel elaboration was requested + if e.Threads > 0 { + return e.ExecuteParallelHTTP(p, reqURL) + } + + var requestNumber int + + result := &Result{ + Matches: make(map[string]interface{}), + Extractions: make(map[string]interface{}), + historyData: make(map[string]interface{}), + } + + dynamicvalues := make(map[string]interface{}) + + // verify if the URL is already being processed + if e.HasGenerator(reqURL) { + return result + } + + remaining := e.GetRequestCount() + e.CreateGenerator(reqURL) + + for e.Next(reqURL) { + requestNumber++ + result.Lock() + httpRequest, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) + payloads, _ := e.GetPayloadsValues(reqURL) + result.Unlock() + // ignore the error due to the base request having null paylods + if err == requests.ErrNoPayload { + // pass through + } else if err != nil { + result.Error = err + p.Drop(remaining) + } else { + e.ratelimiter.Take() + // If the request was built correctly then execute it + format := "%s_" + strconv.Itoa(requestNumber) + err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) + if err != nil { + result.Error = errors.Wrap(err, "could not handle http request") + p.Drop(remaining) + e.traceLog.Request(e.template.ID, reqURL, "http", err) + } else { + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } + } + p.Update() + + // Check if has to stop processing at first valid result + if e.stopAtFirstMatch && result.GotResults { + p.Drop(remaining) + break + } + + // move always forward with requests + e.Increment(reqURL) + remaining-- + } + gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) + return result +} + +func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { + // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given + if e.options.Options.RandomAgent { + // nolint:errcheck // ignoring error + e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) + } + + e.setCustomHeaders(request) + + var ( + resp *http.Response + err error + dumpedRequest []byte + fromcache bool + ) + + if e.debug || e.pf != nil { + dumpedRequest, err = requests.Dump(request, reqURL) + if err != nil { + return err + } + } + + if e.debug { + gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID) + fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) + } + + timeStart := time.Now() + + if request.Pipeline { + resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) + if err != nil { + if resp != nil { + resp.Body.Close() + } + e.traceLog.Request(e.template.ID, reqURL, "http", err) + return err + } + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } else if request.Unsafe { + // rawhttp + // burp uses "\r\n" as new line character + request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") + options := e.rawHTTPClient.Options + options.AutomaticContentLength = request.AutomaticContentLengthHeader + options.AutomaticHostHeader = request.AutomaticHostHeader + options.FollowRedirects = request.FollowRedirects + resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options) + if err != nil { + if resp != nil { + resp.Body.Close() + } + e.traceLog.Request(e.template.ID, reqURL, "http", err) + return err + } + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } else { + // if nuclei-project is available check if the request was already sent previously + if e.pf != nil { + // if unavailable fail silently + fromcache = true + // nolint:bodyclose // false positive the response is generated at runtime + resp, err = e.pf.Get(dumpedRequest) + if err != nil { + fromcache = false + } + } + + // retryablehttp + if resp == nil { + resp, err = e.httpClient.Do(request.Request) + if err != nil { + if resp != nil { + resp.Body.Close() + } + e.traceLog.Request(e.template.ID, reqURL, "http", err) + return err + } + e.traceLog.Request(e.template.ID, reqURL, "http", nil) + } + } + + duration := time.Since(timeStart) + + // Dump response - Step 1 - Decompression not yet handled + var dumpedResponse []byte + if e.debug { + var dumpErr error + dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) + if dumpErr != nil { + return errors.Wrap(dumpErr, "could not dump http response") + } + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + _, copyErr := io.Copy(ioutil.Discard, resp.Body) + if copyErr != nil { + resp.Body.Close() + return copyErr + } + + resp.Body.Close() + + return errors.Wrap(err, "could not read http body") + } + + resp.Body.Close() + + // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request + // so in case we have to manually do it + dataOrig := data + data, err = requests.HandleDecompression(request, data) + if err != nil { + return errors.Wrap(err, "could not decompress http body") + } + + // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) + if e.debug { + dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) + gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) + fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) + } + + // if nuclei-project is enabled store the response if not previously done + if e.pf != nil && !fromcache { + err := e.pf.Set(dumpedRequest, resp, data) + if err != nil { + return errors.Wrap(err, "could not store in project file") + } + } + + // Convert response body from []byte to string with zero copy + body := unsafeToString(data) + + headers := headersToString(resp.Header) + + var matchData map[string]interface{} + if payloads != nil { + matchData = generators.MergeMaps(result.historyData, payloads) + } + + // store for internal purposes the DSL matcher data + // hardcode stopping storing data after defaultMaxHistorydata items + if len(result.historyData) < defaultMaxHistorydata { + result.Lock() + // update history data with current reqURL and hostname + result.historyData["reqURL"] = reqURL + if parsed, err := url.Parse(reqURL); err == nil { + result.historyData["Hostname"] = parsed.Host + } + result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) + if payloads == nil { + // merge them to history data + result.historyData = generators.MergeMaps(result.historyData, payloads) + } + result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) + + // complement match data with new one if necessary + matchData = generators.MergeMaps(matchData, result.historyData) + result.Unlock() + } + + matcherCondition := e.GetMatchersCondition() + for _, matcher := range e.Matchers { + // Check if the matcher matched + if !matcher.Match(resp, body, headers, duration, matchData) { + // If the condition is AND we haven't matched, try next request. + if matcherCondition == matchers.ANDCondition { + return nil + } + } else { + // If the matcher has matched, and its an OR + // write the first output then move to next matcher. + if matcherCondition == matchers.ORCondition { + result.Lock() + result.Matches[matcher.Name] = nil + // probably redundant but ensures we snapshot current payload values when matchers are valid + result.Meta = request.Meta + result.GotResults = true + result.Unlock() + e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL) + } + } + } + + // All matchers have successfully completed so now start with the + // next task which is extraction of input from matchers. + var extractorResults, outputExtractorResults []string + + for _, extractor := range e.Extractors { + for match := range extractor.Extract(resp, body, headers) { + if _, ok := dynamicvalues[extractor.Name]; !ok { + dynamicvalues[extractor.Name] = match + } + + extractorResults = append(extractorResults, match) + + if !extractor.Internal { + outputExtractorResults = append(outputExtractorResults, match) + } + } + // probably redundant but ensures we snapshot current payload values when extractors are valid + result.Lock() + result.Meta = request.Meta + result.Extractions[extractor.Name] = extractorResults + result.Unlock() + } + + // Write a final string of output if matcher type is + // AND or if we have extractors for the mechanism too. + if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { + e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL) + result.Lock() + result.GotResults = true + result.Unlock() + } + + return nil +} diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index e801c0e7f..0a1252e5e 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -1,13 +1,8 @@ package requests import ( - "fmt" - "net" - "net/http" - "net/url" "regexp" - "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/generators" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/rawhttp" @@ -63,92 +58,3 @@ type HTTPRequest struct { Httpclient *retryablehttp.Client PipelineClient *rawhttp.PipelineClient } - -func setHeader(req *http.Request, name, value string) { - // Set some headers only if the header wasn't supplied by the user - if _, ok := req.Header[name]; !ok { - req.Header.Set(name, value) - } -} - -// baseURLWithTemplatePrefs returns the url for BaseURL keeping -// the template port and path preference -func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { - // template port preference over input URL port - // template has port - hasPort := len(urlWithPortRgx.FindStringSubmatch(data)) > 0 - if hasPort { - // check if also the input contains port, in this case extracts the url - if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil { - parsedURL.Host = hostname - } - } - return parsedURL.String() -} - -// Next returns the next generator by URL -func (r *BulkHTTPRequest) Next(reqURL string) bool { - return r.gsfm.Next(reqURL) -} - -// Position returns the current generator's position by URL -func (r *BulkHTTPRequest) Position(reqURL string) int { - return r.gsfm.Position(reqURL) -} - -// Reset resets the generator by URL -func (r *BulkHTTPRequest) Reset(reqURL string) { - r.gsfm.Reset(reqURL) -} - -// Current returns the current generator by URL -func (r *BulkHTTPRequest) Current(reqURL string) string { - return r.gsfm.Current(reqURL) -} - -// Total is the total number of requests -func (r *BulkHTTPRequest) Total() int { - return r.gsfm.Total() -} - -// Increment increments the processed request -func (r *BulkHTTPRequest) Increment(reqURL string) { - r.gsfm.Increment(reqURL) -} - -// GetPayloadsValues for the specified URL -func (r *BulkHTTPRequest) GetPayloadsValues(reqURL string) (map[string]interface{}, error) { - payloadProcessedValues := make(map[string]interface{}) - payloadsFromTemplate := r.gsfm.Value(reqURL) - for k, v := range payloadsFromTemplate { - kexp := v.(string) - // if it doesn't containing markups, we just continue - if !hasMarker(kexp) { - payloadProcessedValues[k] = v - continue - } - // attempts to expand expressions - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions()) - if err != nil { - // it is a simple literal payload => proceed with literal value - payloadProcessedValues[k] = v - continue - } - // it is an expression - try to solve it - expValue, err := compiled.Evaluate(payloadsFromTemplate) - if err != nil { - // an error occurred => proceed with literal value - payloadProcessedValues[k] = v - continue - } - payloadProcessedValues[k] = fmt.Sprint(expValue) - } - var err error - if len(payloadProcessedValues) == 0 { - err = ErrNoPayload - } - return payloadProcessedValues, err -} - -// ErrNoPayload error to avoid the additional base null request -var ErrNoPayload = fmt.Errorf("no payload found") From e8a17e18ca1203f5f142bdcbf403f47398f221fa Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 22:52:33 +0530 Subject: [PATCH 28/92] Fixed payload generators bug --- .../protocols/common/generators/generators.go | 129 ++++++++++-------- .../common/generators/generators_test.go | 34 ++++- 2 files changed, 101 insertions(+), 62 deletions(-) diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 27bfcab04..4299a2895 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -2,6 +2,10 @@ package generators +import ( + "errors" +) + // Generator is the generator struct for generating payloads type Generator struct { Type Type @@ -33,6 +37,20 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) { if err != nil { return nil, err } + + // Validate the payload types + if Type == Sniper && len(compiled) > 1 { + return nil, errors.New("cannot use more than one payload set in sniper") + } + if Type == PitchFork { + var totalLength int + for v := range compiled { + if totalLength != 0 && totalLength != len(v) { + return nil, errors.New("pitchfork payloads must be of equal number") + } + totalLength = len(v) + } + } return &Generator{Type: Type, payloads: compiled}, nil } @@ -41,6 +59,7 @@ type Iterator struct { Type Type position int msbIterator int + total int payloads []*payloadIterator } @@ -51,16 +70,22 @@ func (g *Generator) NewIterator() *Iterator { for name, values := range g.payloads { payloads = append(payloads, &payloadIterator{name: name, values: values}) } - return &Iterator{Type: g.Type, payloads: payloads} + iterator := &Iterator{ + Type: g.Type, + payloads: payloads, + } + iterator.total = iterator.Total() + return iterator } -// Next returns true if there are more inputs in iterator -func (i *Iterator) Next() bool { - if i.position >= i.Total() { - return false +// Reset resets the iterator back to its initial value +func (i *Iterator) Reset() { + i.position = 0 + i.msbIterator = 0 + + for _, payload := range i.payloads { + payload.resetPosition() } - i.position++ - return true } //Total returns the amount of input combinations available @@ -68,28 +93,22 @@ func (i *Iterator) Total() int { count := 0 switch i.Type { case Sniper: - for _, p := range i.payloads { - if p.Total() > count { - count = p.Total() - } - } + count = len(i.payloads[0].values) case PitchFork: for _, p := range i.payloads { - if p.Total() > count { - count = p.Total() - } + count = len(p.values) } case ClusterBomb: count = 1 for _, p := range i.payloads { - count = count * p.Total() + count = count * len(p.values) } } return count } // Value returns the next value for an iterator -func (i *Iterator) Value() map[string]interface{} { +func (i *Iterator) Value() (map[string]interface{}, bool) { switch i.Type { case Sniper: return i.sniperValue() @@ -103,35 +122,37 @@ func (i *Iterator) Value() map[string]interface{} { } // sniperValue returns a list of all payloads for the iterator -func (i *Iterator) sniperValue() map[string]interface{} { - values := make(map[string]interface{}, len(i.payloads)) +func (i *Iterator) sniperValue() (map[string]interface{}, bool) { + values := make(map[string]interface{}, 1) - for _, p := range i.payloads { - if !p.Next() { - p.ResetPosition() - } - values[p.name] = p.Value() - p.IncrementPosition() + payload := i.payloads[0] + if !payload.next() { + return nil, false } - return values + values[payload.name] = payload.value() + payload.incrementPosition() + return values, true } // pitchforkValue returns a map of keyword:value pairs in same index -func (i *Iterator) pitchforkValue() map[string]interface{} { +func (i *Iterator) pitchforkValue() (map[string]interface{}, bool) { values := make(map[string]interface{}, len(i.payloads)) for _, p := range i.payloads { - if !p.Next() { - p.ResetPosition() + if !p.next() { + return nil, false } - values[p.name] = p.Value() - p.IncrementPosition() + values[p.name] = p.value() + p.incrementPosition() } - return values + return values, true } // clusterbombValue returns a combination of all input pairs in key:value format. -func (i *Iterator) clusterbombValue() map[string]interface{} { +func (i *Iterator) clusterbombValue() (map[string]interface{}, bool) { + if i.position >= i.total { + return nil, false + } values := make(map[string]interface{}, len(i.payloads)) // Should we signal the next InputProvider in the slice to increment @@ -139,10 +160,10 @@ func (i *Iterator) clusterbombValue() map[string]interface{} { first := true for index, p := range i.payloads { if signalNext { - p.IncrementPosition() + p.incrementPosition() signalNext = false } - if !p.Next() { + if !p.next() { // No more inputs in this inputprovider if index == i.msbIterator { // Reset all previous wordlists and increment the msb counter @@ -151,25 +172,26 @@ func (i *Iterator) clusterbombValue() map[string]interface{} { // Start again return i.clusterbombValue() } - p.ResetPosition() + p.resetPosition() signalNext = true } - values[p.name] = p.Value() + values[p.name] = p.value() if first { - p.IncrementPosition() + p.incrementPosition() first = false } } - return values + i.position++ + return values, true } func (i *Iterator) clusterbombIteratorReset() { for index, p := range i.payloads { if index < i.msbIterator { - p.ResetPosition() + p.resetPosition() } if index == i.msbIterator { - p.IncrementPosition() + p.incrementPosition() } } } @@ -181,27 +203,22 @@ type payloadIterator struct { values []string } -// Next returns true if there are more values in payload iterator -func (i *payloadIterator) Next() bool { - if i.index >= i.Total() { - return false - } - return true +// next returns true if there are more values in payload iterator +func (i *payloadIterator) next() bool { + return i.index < len(i.values) } -func (i *payloadIterator) ResetPosition() { +// resetPosition resets the position of the payload iterator +func (i *payloadIterator) resetPosition() { i.index = 0 } -func (i *payloadIterator) IncrementPosition() { +// incrementPosition increments the position of the payload iterator +func (i *payloadIterator) incrementPosition() { i.index++ } -func (i *payloadIterator) Value() string { - value := i.values[i.index] - return value -} - -func (i *payloadIterator) Total() int { - return len(i.values) +// value returns the value of the payload at an index +func (i *payloadIterator) value() string { + return i.values[i.index] } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index e27f4b85f..93843c856 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -14,9 +14,12 @@ func TestSniperGenerator(t *testing.T) { iterator := generator.NewIterator() count := 0 - for iterator.Next() { + for { + value, ok := iterator.Value() + if !ok { + break + } count++ - value := iterator.Value() require.Contains(t, usernames, value["username"], "Could not get correct sniper") } require.Equal(t, len(usernames), count, "could not get correct sniper counts") @@ -31,9 +34,12 @@ func TestPitchforkGenerator(t *testing.T) { iterator := generator.NewIterator() count := 0 - for iterator.Next() { + for { + value, ok := iterator.Value() + if !ok { + break + } count++ - value := iterator.Value() require.Contains(t, usernames, value["username"], "Could not get correct pitchfork username") require.Contains(t, passwords, value["password"], "Could not get correct pitchfork password") } @@ -49,9 +55,25 @@ func TestClusterbombGenerator(t *testing.T) { iterator := generator.NewIterator() count := 0 - for iterator.Next() { + for { + value, ok := iterator.Value() + if !ok { + break + } + count++ + require.Contains(t, usernames, value["username"], "Could not get correct clusterbomb username") + require.Contains(t, passwords, value["password"], "Could not get correct clusterbomb password") + } + require.Equal(t, 3, count, "could not get correct clusterbomb counts") + + iterator.Reset() + count = 0 + for { + value, ok := iterator.Value() + if !ok { + break + } count++ - value := iterator.Value() require.Contains(t, usernames, value["username"], "Could not get correct clusterbomb username") require.Contains(t, passwords, value["password"], "Could not get correct clusterbomb password") } From 2ded647536e95e2866547d546c25c678f8df89a2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 26 Dec 2020 22:57:40 +0530 Subject: [PATCH 29/92] Initial generators work started for http + payloads --- v2/pkg/protocols/http/build_request.go | 191 +++++++++++--------- v2/pkg/protocols/http/build_request_test.go | 57 ++++++ v2/pkg/protocols/http/http.go | 1 + v2/pkg/protocols/http/request.go | 25 +-- 4 files changed, 163 insertions(+), 111 deletions(-) create mode 100644 v2/pkg/protocols/http/build_request_test.go diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ba72280b7..ac376b655 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -1,29 +1,89 @@ package http import ( - "context" "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "net/url" "regexp" - "strings" - "time" - "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/pkg/protcols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" - "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" ) var urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) -// MakeHTTPRequest makes the HTTP request -func (r *Request) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) { +// requestGenerator generates requests sequentially based on various +// configurations for a http request template. +// +// If payload values are present, an iterator is created for the payload +// values. Paths and Raw requests are supported as base input, so +// it will automatically select between them based on the template. +type requestGenerator struct { + currentIndex int + request *Request + payloadIterator *generators.Iterator +} + +// newGenerator creates a new request generator instance +func (r *Request) newGenerator() *requestGenerator { + generator := &requestGenerator{request: r} + + if len(r.Payloads) > 0 { + generator.payloadIterator = r.generator.NewIterator() + } + return generator +} + +// nextValue returns the next path or the next raw request depending on user input +// It returns false if all the inputs have been exhausted by the generator instance. +func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) { + // If we have paths, return the next path. + if len(r.request.Path) > 0 && r.currentIndex < len(r.request.Path) { + if item := r.request.Path[r.currentIndex]; item != "" { + r.currentIndex++ + return item, nil, true + } + } + + // If we have raw requests, start with the request at current index. + // If we are not at the start, then check if the iterator for payloads + // has finished if there are any. + // + // If the iterator has finished for the current raw request + // then reset it and move on to the next value, otherwise use the last request. + if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) { + if r.payloadIterator != nil { + payload, ok := r.payloadIterator.Value() + if !ok { + r.currentIndex++ + r.payloadIterator.Reset() + + // No more payloads request for us now. + if len(r.request.Raw) == r.currentIndex { + return "", nil, false + } + if item := r.request.Raw[r.currentIndex]; item != "" { + newPayload, ok := r.payloadIterator.Value() + return item, newPayload, ok + } + return "", nil, false + } + fmt.Printf("index-last: %v\n", r.currentIndex) + return r.request.Raw[r.currentIndex], payload, true + } + if item := r.request.Raw[r.currentIndex]; item != "" { + r.currentIndex++ + return item, nil, true + } + } + return "", nil, false +} + +/* +// Make creates a http request for the provided input. +// It returns io.EOF as error when all the requests have been exhausted. +func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*HTTPRequest, error) { + data, ok := r.nextValue() + if !ok { + return nil, io.EOF + } ctx := context.Background() parsed, err := url.Parse(baseURL) @@ -38,17 +98,34 @@ func (r *Request) MakeHTTPRequest(baseURL string, dynamicValues map[string]inter "Hostname": hostname, }) - // if data contains \n it's a raw request + // If data contains \n it's a raw request, process it like that. Else + // continue with the template based request flow. if strings.Contains(data, "\n") { return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values) } return r.makeHTTPRequestFromModel(ctx, data, values) } +// baseURLWithTemplatePrefs returns the url for BaseURL keeping +// the template port and path preference +func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { + // template port preference over input URL port + // template has port + hasPort := len(urlWithPortRegex.FindStringSubmatch(data)) > 0 + if hasPort { + // check if also the input contains port, in this case extracts the url + if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil { + parsedURL.Host = hostname + } + } + return parsedURL.String() +} + +/* + // MakeHTTPRequestFromModel creates a *http.Request from a request template func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) { - replacer := newReplacer(values) - URL := replacer.Replace(data) + URL := replacer.New(values).Replace(data) // Build a request on the specified URL req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil) @@ -63,40 +140,20 @@ func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, val return &HTTPRequest{Request: request}, nil } -// InitGenerator initializes the generator -func (r *Request) InitGenerator() { - r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw) -} - -// CreateGenerator creates the generator -func (r *Request) CreateGenerator(reqURL string) { - r.gsfm.Add(reqURL) -} - -// HasGenerator check if an URL has a generator -func (r *Request) HasGenerator(reqURL string) bool { - return r.gsfm.Has(reqURL) -} - -// ReadOne reads and return a generator by URL -func (r *Request) ReadOne(reqURL string) { - r.gsfm.ReadOne(reqURL) -} - // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *Request) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) { // Add trailing line data += "\n" + // If we have payloads, handle them by creating a generator if len(r.Payloads) > 0 { r.gsfm.InitOrSkip(baseURL) r.ReadOne(baseURL) - payloads, err := r.GetPayloadsValues(baseURL) + payloads, err := r.getPayloadValues(baseURL) if err != nil { return nil, err } - return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) } @@ -199,7 +256,7 @@ func (r *Request) fillRequest(req *http.Request, values map[string]interface{}) if len(r.Raw) > 0 { return retryablehttp.FromRequest(req) } - setHeader(req, "Accept", "*/*") + //setHeader(req, "Accept", "") setHeader(req, "Accept-Language", "en") return retryablehttp.FromRequest(req) @@ -212,55 +269,12 @@ func setHeader(req *http.Request, name, value string) { } } -// baseURLWithTemplatePrefs returns the url for BaseURL keeping -// the template port and path preference -func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { - // template port preference over input URL port - // template has port - hasPort := len(urlWithPortRegex.FindStringSubmatch(data)) > 0 - if hasPort { - // check if also the input contains port, in this case extracts the url - if hostname, _, err := net.SplitHostPort(parsedURL.Host); err == nil { - parsedURL.Host = hostname - } - } - return parsedURL.String() -} -// Next returns the next generator by URL -func (r *Request) Next(reqURL string) bool { - return r.gsfm.Next(reqURL) -} - -// Position returns the current generator's position by URL -func (r *Request) Position(reqURL string) int { - return r.gsfm.Position(reqURL) -} - -// Reset resets the generator by URL -func (r *Request) Reset(reqURL string) { - r.gsfm.Reset(reqURL) -} - -// Current returns the current generator by URL -func (r *Request) Current(reqURL string) string { - return r.gsfm.Current(reqURL) -} - -// Total is the total number of requests -func (r *Request) Total() int { - return r.gsfm.Total() -} - -// Increment increments the processed request -func (r *Request) Increment(reqURL string) { - r.gsfm.Increment(reqURL) -} - -// GetPayloadsValues for the specified URL -func (r *Request) GetPayloadsValues(reqURL string) (map[string]interface{}, error) { +// getPayloadValues returns current payload values for a request +func (r *Request) getPayloadValues(reqURL string) (map[string]interface{}, error) { payloadProcessedValues := make(map[string]interface{}) payloadsFromTemplate := r.gsfm.Value(reqURL) + for k, v := range payloadsFromTemplate { kexp := v.(string) // if it doesn't containing markups, we just continue @@ -293,3 +307,4 @@ func (r *Request) GetPayloadsValues(reqURL string) (map[string]interface{}, erro // ErrNoPayload error to avoid the additional base null request var ErrNoPayload = fmt.Errorf("no payload found") +*/ diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go new file mode 100644 index 000000000..5793a23b6 --- /dev/null +++ b/v2/pkg/protocols/http/build_request_test.go @@ -0,0 +1,57 @@ +package http + +import ( + "fmt" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/stretchr/testify/require" +) + +func TestRequestGeneratorClusterSingle(t *testing.T) { + var err error + + req := &Request{ + Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, + attackType: generators.ClusterBomb, + Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, + } + req.generator, err = generators.New(req.Payloads, req.attackType) + require.Nil(t, err, "could not create generator") + + generator := req.newGenerator() + var payloads []map[string]interface{} + for { + raw, data, ok := generator.nextValue() + if !ok { + break + } + payloads = append(payloads, data) + fmt.Printf("%v %v\n", raw, data) + } + require.Equal(t, 9, len(payloads), "Could not get correct number of payloads") +} + +func TestRequestGeneratorClusterMultipleRaw(t *testing.T) { + var err error + + req := &Request{ + Payloads: map[string]interface{}{"username": []string{"admin", "tomcat", "manager"}, "password": []string{"password", "test", "secret"}}, + attackType: generators.ClusterBomb, + Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, + } + req.generator, err = generators.New(req.Payloads, req.attackType) + require.Nil(t, err, "could not create generator") + + generator := req.newGenerator() + var payloads []map[string]interface{} + for { + raw, data, ok := generator.nextValue() + if !ok { + break + } + payloads = append(payloads, data) + fmt.Printf("%v %v\n", raw, data) + } + require.Equal(t, 18, len(payloads), "Could not get correct number of payloads") +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 736972824..0c6be24b9 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -52,5 +52,6 @@ type Request struct { Race bool `yaml:"race"` attackType generators.Type + generator *generators.Generator // optional, only enabled when using payloads options *protocols.ExecuterOptions } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index f8639673d..e50567502 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -1,28 +1,6 @@ package http -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/corpix/uarand" - "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/pkg/protcols/common/generators" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/rawhttp" - "github.com/remeh/sizedwaitgroup" -) - +/* func (e *Request) ExecuteRaceRequest(reqURL string) *Result { result := &Result{ Matches: make(map[string]interface{}), @@ -498,3 +476,4 @@ func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynam return nil } +*/ From 40d56553280550c6a7a046c8c9cb7c0ca4a0a7ff Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 28 Dec 2020 01:33:50 +0530 Subject: [PATCH 30/92] HTTP request building workflow changes --- v2/pkg/protocols/common/generators/maps.go | 38 +++ v2/pkg/protocols/common/replacer/replacer.go | 11 +- v2/pkg/protocols/http/build_request.go | 124 +++++---- v2/pkg/protocols/http/build_request_test.go | 23 +- v2/pkg/protocols/http/http.go | 48 +++- v2/pkg/requests/bulk-http-request.go | 60 ---- v2/pkg/requests/generator.go | 273 ------------------- v2/pkg/requests/util.go | 10 - 8 files changed, 173 insertions(+), 414 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/maps.go delete mode 100644 v2/pkg/requests/bulk-http-request.go delete mode 100644 v2/pkg/requests/generator.go diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go new file mode 100644 index 000000000..da78af9d3 --- /dev/null +++ b/v2/pkg/protocols/common/generators/maps.go @@ -0,0 +1,38 @@ +package generators + +import "strings" + +// MergeMaps merges two maps into a new map +func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} { + m := make(map[string]interface{}, len(m1)+len(m2)) + for k, v := range m1 { + m[k] = v + } + for k, v := range m2 { + m[k] = v + } + return m +} + +// CopyMap creates a new copy of an existing map +func CopyMap(originalMap map[string]interface{}) map[string]interface{} { + newMap := make(map[string]interface{}) + for key, value := range originalMap { + newMap[key] = value + } + return newMap +} + +// CopyMapWithDefaultValue creates a new copy of an existing map and set a default value +func CopyMapWithDefaultValue(originalMap map[string][]string, defaultValue interface{}) map[string]interface{} { + newMap := make(map[string]interface{}) + for key := range originalMap { + newMap[key] = defaultValue + } + return newMap +} + +// TrimDelimiters removes trailing brackets +func TrimDelimiters(s string) string { + return strings.TrimSuffix(strings.TrimPrefix(s, "{{"), "}}") +} diff --git a/v2/pkg/protocols/common/replacer/replacer.go b/v2/pkg/protocols/common/replacer/replacer.go index 29f4aece5..06e12c36e 100644 --- a/v2/pkg/protocols/common/replacer/replacer.go +++ b/v2/pkg/protocols/common/replacer/replacer.go @@ -5,10 +5,11 @@ import ( "strings" ) +// Payload marker constants const ( - markerGeneral = "§" - markerParenthesisOpen = "{{" - markerParenthesisClose = "}}" + MarkerGeneral = "§" + MarkerParenthesisOpen = "{{" + MarkerParenthesisClose = "}}" ) // New creates a new replacer structure for values replacement on the fly. @@ -19,11 +20,11 @@ func New(values map[string]interface{}) *strings.Replacer { valueStr := fmt.Sprintf("%s", val) replacerItems = append(replacerItems, - fmt.Sprintf("%s%s%s", markerParenthesisOpen, key, markerParenthesisClose), + fmt.Sprintf("%s%s%s", MarkerParenthesisOpen, key, MarkerParenthesisClose), valueStr, ) replacerItems = append(replacerItems, - fmt.Sprintf("%s%s%s", markerGeneral, key, markerGeneral), + fmt.Sprintf("%s%s%s", MarkerGeneral, key, MarkerGeneral), valueStr, ) } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index ac376b655..256031cdb 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -1,13 +1,30 @@ package http import ( + "context" "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" "regexp" + "strings" + "time" + "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/retryablehttp-go" ) -var urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) +var ( + urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) + templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) +) // requestGenerator generates requests sequentially based on various // configurations for a http request template. @@ -65,7 +82,6 @@ func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) { } return "", nil, false } - fmt.Printf("index-last: %v\n", r.currentIndex) return r.request.Raw[r.currentIndex], payload, true } if item := r.request.Raw[r.currentIndex]; item != "" { @@ -76,11 +92,18 @@ func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) { return "", nil, false } -/* +// generatedRequest is a single wrapped generated request for a template request +type generatedRequest struct { + original *Request + rawRequest *raw.Request + meta map[string]interface{} + request *retryablehttp.Request +} + // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. -func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*HTTPRequest, error) { - data, ok := r.nextValue() +func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) { + data, payloads, ok := r.nextValue() if !ok { return nil, io.EOF } @@ -92,7 +115,6 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } hostname := parsed.Host - values := generators.MergeMaps(dynamicValues, map[string]interface{}{ "BaseURL": baseURLWithTemplatePrefs(data, parsed), "Hostname": hostname, @@ -101,7 +123,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa // If data contains \n it's a raw request, process it like that. Else // continue with the template based request flow. if strings.Contains(data, "\n") { - return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values) + return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values, payloads) } return r.makeHTTPRequestFromModel(ctx, data, values) } @@ -121,14 +143,12 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { return parsedURL.String() } -/* - // MakeHTTPRequestFromModel creates a *http.Request from a request template -func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) { +func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*generatedRequest, error) { URL := replacer.New(values).Replace(data) // Build a request on the specified URL - req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil) + req, err := http.NewRequestWithContext(ctx, r.request.Method, URL, nil) if err != nil { return nil, err } @@ -137,31 +157,27 @@ func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, val if err != nil { return nil, err } - return &HTTPRequest{Request: request}, nil + return &generatedRequest{request: request}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request -func (r *Request) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) { +func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { // Add trailing line data += "\n" - // If we have payloads, handle them by creating a generator - if len(r.Payloads) > 0 { - r.gsfm.InitOrSkip(baseURL) - r.ReadOne(baseURL) - - payloads, err := r.getPayloadValues(baseURL) + // If we have payloads, handle them by evaluating them at runtime. + if len(r.request.Payloads) > 0 { + finalPayloads, err := r.getPayloadValues(baseURL, payloads) if err != nil { return nil, err } - return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) + return r.handleRawWithPaylods(ctx, data, baseURL, values, finalPayloads) } - - // otherwise continue with normal flow return r.handleRawWithPaylods(ctx, data, baseURL, values, nil) } -func (r *Request) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) { +// handleRawWithPaylods handles raw requests along with paylaods +func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, genValues map[string]interface{}) (*generatedRequest, error) { baseValues := generators.CopyMap(values) finValues := generators.MergeMaps(baseValues, genValues) @@ -169,12 +185,10 @@ func (r *Request) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL rawRequest = replacer.New(finValues).Replace(rawRequest) dynamicValues := make(map[string]interface{}) - // find all potentials tokens between {{}} - var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`) - for _, match := range re.FindAllString(rawRequest, -1) { + for _, match := range templateExpressionRegex.FindAllString(rawRequest, -1) { // check if the match contains a dynamic variable expr := generators.TrimDelimiters(match) - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions()) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) if err != nil { return nil, err @@ -189,40 +203,37 @@ func (r *Request) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL // Replacer dynamic values if any in raw request and parse it rawRequest = replacer.New(dynamicValues).Replace(rawRequest) - rawRequestData, err := raw.Parse(rawRequest, baseURL, r.Unsafe) + rawRequestData, err := raw.Parse(rawRequest, baseURL, r.request.Unsafe) if err != nil { return nil, err } // rawhttp - if r.Unsafe { - unsafeReq := &HTTPRequest{ - RawRequest: rawRequest, - Meta: genValues, - AutomaticHostHeader: !r.DisableAutoHostname, - AutomaticContentLengthHeader: !r.DisableAutoContentLength, - Unsafe: true, - FollowRedirects: r.Redirects, + if r.request.Unsafe { + unsafeReq := &generatedRequest{ + rawRequest: rawRequestData, + meta: genValues, + original: r.request, } return unsafeReq, nil } // retryablehttp var body io.ReadCloser - body = ioutil.NopCloser(strings.NewReader(rawRequest.Data)) - if r.Race { + body = ioutil.NopCloser(strings.NewReader(rawRequestData.Data)) + if r.request.Race { // More or less this ensures that all requests hit the endpoint at the same approximated time // Todo: sync internally upon writing latest request byte body = race.NewOpenGateWithTimeout(body, time.Duration(2)*time.Second) } - req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body) + req, err := http.NewRequestWithContext(ctx, rawRequestData.Method, rawRequestData.FullURL, body) if err != nil { return nil, err } // copy headers - for key, value := range rawRequest.Headers { + for key, value := range rawRequestData.Headers { req.Header[key] = []string{value} } @@ -230,33 +241,33 @@ func (r *Request) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL if err != nil { return nil, err } - - return &HTTPRequest{Request: request, Meta: genValues}, nil + return &generatedRequest{request: request, meta: genValues}, nil } -func (r *Request) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { - replacer := replacer.New(values) +// fillRequest fills various headers in the request with values +func (r *requestGenerator) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) { // Set the header values requested - for header, value := range r.Headers { + replacer := replacer.New(values) + for header, value := range r.request.Headers { req.Header[header] = []string{replacer.Replace(value)} } // In case of multiple threads the underlying connection should remain open to allow reuse - if r.Threads <= 0 && req.Header.Get("Connection") == "" { + if r.request.Threads <= 0 && req.Header.Get("Connection") == "" { req.Close = true } // Check if the user requested a request body - if r.Body != "" { - req.Body = ioutil.NopCloser(strings.NewReader(r.Body)) + if r.request.Body != "" { + req.Body = ioutil.NopCloser(strings.NewReader(r.request.Body)) } setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)") // raw requests are left untouched - if len(r.Raw) > 0 { + if len(r.request.Raw) > 0 { return retryablehttp.FromRequest(req) } - //setHeader(req, "Accept", "") + setHeader(req, "Accept", "*/*") setHeader(req, "Accept-Language", "en") return retryablehttp.FromRequest(req) @@ -269,28 +280,26 @@ func setHeader(req *http.Request, name, value string) { } } - // getPayloadValues returns current payload values for a request -func (r *Request) getPayloadValues(reqURL string) (map[string]interface{}, error) { +func (r *requestGenerator) getPayloadValues(reqURL string, templatePayloads map[string]interface{}) (map[string]interface{}, error) { payloadProcessedValues := make(map[string]interface{}) - payloadsFromTemplate := r.gsfm.Value(reqURL) - for k, v := range payloadsFromTemplate { + for k, v := range templatePayloads { kexp := v.(string) // if it doesn't containing markups, we just continue - if !hasMarker(kexp) { + if !strings.Contains(kexp, replacer.MarkerParenthesisOpen) || strings.Contains(kexp, replacer.MarkerParenthesisClose) || strings.Contains(kexp, replacer.MarkerGeneral) { payloadProcessedValues[k] = v continue } // attempts to expand expressions - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, generators.HelperFunctions()) + compiled, err := govaluate.NewEvaluableExpressionWithFunctions(kexp, dsl.HelperFunctions()) if err != nil { // it is a simple literal payload => proceed with literal value payloadProcessedValues[k] = v continue } // it is an expression - try to solve it - expValue, err := compiled.Evaluate(payloadsFromTemplate) + expValue, err := compiled.Evaluate(templatePayloads) if err != nil { // an error occurred => proceed with literal value payloadProcessedValues[k] = v @@ -307,4 +316,3 @@ func (r *Request) getPayloadValues(reqURL string) (map[string]interface{}, error // ErrNoPayload error to avoid the additional base null request var ErrNoPayload = fmt.Errorf("no payload found") -*/ diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 5793a23b6..846b8896c 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -1,13 +1,28 @@ package http import ( - "fmt" "testing" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/stretchr/testify/require" ) +func TestRequestGeneratorPaths(t *testing.T) { + req := &Request{ + Path: []string{"{{BaseURL}}/test", "{{BaseURL}}/test.php"}, + } + generator := req.newGenerator() + var payloads []string + for { + raw, _, ok := generator.nextValue() + if !ok { + break + } + payloads = append(payloads, raw) + } + require.Equal(t, req.Path, payloads, "Could not get correct paths") +} + func TestRequestGeneratorClusterSingle(t *testing.T) { var err error @@ -22,12 +37,11 @@ func TestRequestGeneratorClusterSingle(t *testing.T) { generator := req.newGenerator() var payloads []map[string]interface{} for { - raw, data, ok := generator.nextValue() + _, data, ok := generator.nextValue() if !ok { break } payloads = append(payloads, data) - fmt.Printf("%v %v\n", raw, data) } require.Equal(t, 9, len(payloads), "Could not get correct number of payloads") } @@ -46,12 +60,11 @@ func TestRequestGeneratorClusterMultipleRaw(t *testing.T) { generator := req.newGenerator() var payloads []map[string]interface{} for { - raw, data, ok := generator.nextValue() + _, data, ok := generator.nextValue() if !ok { break } payloads = append(payloads, data) - fmt.Printf("%v %v\n", raw, data) } require.Equal(t, 18, len(payloads), "Could not get correct number of payloads") } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 0c6be24b9..fb40407ee 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,8 +1,13 @@ package http import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/retryablehttp-go" ) // Request contains a http request to be made from a template @@ -51,7 +56,44 @@ type Request struct { // The minimum number fof requests is determined by threads Race bool `yaml:"race"` - attackType generators.Type - generator *generators.Generator // optional, only enabled when using payloads - options *protocols.ExecuterOptions + // Operators for the current request go here. + *operators.Operators + + options *protocols.ExecuterOptions + attackType generators.Type + generator *generators.Generator // optional, only enabled when using payloads + httpClient *retryablehttp.Client + rawhttpClient *rawhttp.Client +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{ + Threads: r.Threads, + MaxRedirects: r.MaxRedirects, + FollowRedirects: r.Redirects, + }) + if err != nil { + return errors.Wrap(err, "could not get dns client") + } + r.httpClient = client + + if len(r.Raw) > 0 { + r.rawhttpClient = httpclientpool.GetRawHTTP() + } + if r.Operators != nil { + if err := r.Operators.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + } + + if len(r.Payloads) > 0 { + r.attackType = generators.StringToType[r.AttackType] + r.generator, err = generators.New(r.Payloads, r.attackType) + if err != nil { + return errors.Wrap(err, "could not parse payloads") + } + } + r.options = options + return nil } diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go deleted file mode 100644 index 0a1252e5e..000000000 --- a/v2/pkg/requests/bulk-http-request.go +++ /dev/null @@ -1,60 +0,0 @@ -package requests - -import ( - "regexp" - - "github.com/projectdiscovery/nuclei/v2/pkg/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/rawhttp" - retryablehttp "github.com/projectdiscovery/retryablehttp-go" -) - -const ( - two = 2 - three = 3 -) - -var urlWithPortRgx = regexp.MustCompile(`{{BaseURL}}:(\d+)`) - -// GetMatchersCondition returns the condition for the matcher -func (r *BulkHTTPRequest) GetMatchersCondition() matchers.ConditionType { - return r.matchersCondition -} - -// SetMatchersCondition sets the condition for the matcher -func (r *BulkHTTPRequest) SetMatchersCondition(condition matchers.ConditionType) { - r.matchersCondition = condition -} - -// GetAttackType returns the attack -func (r *BulkHTTPRequest) GetAttackType() generators.Type { - return r.attackType -} - -// SetAttackType sets the attack -func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) { - r.attackType = attack -} - -// GetRequestCount returns the total number of requests the YAML rule will perform -func (r *BulkHTTPRequest) GetRequestCount() int64 { - return int64(r.gsfm.Total()) -} - -// HTTPRequest is the basic HTTP request -type HTTPRequest struct { - Request *retryablehttp.Request - RawRequest *RawRequest - Meta map[string]interface{} - - // flags - Unsafe bool - Pipeline bool - AutomaticHostHeader bool - AutomaticContentLengthHeader bool - AutomaticConnectionHeader bool - FollowRedirects bool - Rawclient *rawhttp.Client - Httpclient *retryablehttp.Client - PipelineClient *rawhttp.PipelineClient -} diff --git a/v2/pkg/requests/generator.go b/v2/pkg/requests/generator.go deleted file mode 100644 index 50e80e4a3..000000000 --- a/v2/pkg/requests/generator.go +++ /dev/null @@ -1,273 +0,0 @@ -package requests - -import ( - "sync" - "time" - - "github.com/projectdiscovery/nuclei/v2/pkg/generators" -) - -type GeneratorState int - -const ( - fifteen = 15 - initial GeneratorState = iota - running - done -) - -type Generator struct { - sync.RWMutex - positionPath int - positionRaw int - gchan chan map[string]interface{} - currentGeneratorValue map[string]interface{} - state GeneratorState -} - -type GeneratorFSM struct { - sync.RWMutex - payloads map[string]interface{} - basePayloads map[string][]string - generator func(payloads map[string][]string) (out chan map[string]interface{}) - Generators map[string]*Generator - Type generators.Type - Paths []string - Raws []string -} - -func NewGeneratorFSM(typ generators.Type, payloads map[string]interface{}, paths, raws []string) *GeneratorFSM { - var gsfm GeneratorFSM - gsfm.payloads = payloads - gsfm.Paths = paths - gsfm.Raws = raws - gsfm.Type = typ - - if len(gsfm.payloads) > 0 { - // load payloads if not already done - if gsfm.basePayloads == nil { - gsfm.basePayloads = generators.LoadPayloads(gsfm.payloads) - } - - generatorFunc := generators.SniperGenerator - - switch typ { - case generators.PitchFork: - generatorFunc = generators.PitchforkGenerator - case generators.ClusterBomb: - generatorFunc = generators.ClusterbombGenerator - case generators.Sniper: - generatorFunc = generators.SniperGenerator - } - - gsfm.generator = generatorFunc - } - - gsfm.Generators = make(map[string]*Generator) - - return &gsfm -} - -func (gfsm *GeneratorFSM) Add(key string) { - gfsm.Lock() - defer gfsm.Unlock() - - if _, ok := gfsm.Generators[key]; !ok { - gfsm.Generators[key] = &Generator{state: initial} - } -} - -func (gfsm *GeneratorFSM) Has(key string) bool { - gfsm.RLock() - defer gfsm.RUnlock() - - _, ok := gfsm.Generators[key] - - return ok -} - -func (gfsm *GeneratorFSM) Delete(key string) { - gfsm.Lock() - defer gfsm.Unlock() - - delete(gfsm.Generators, key) -} - -func (gfsm *GeneratorFSM) ReadOne(key string) { - gfsm.RLock() - defer gfsm.RUnlock() - g, ok := gfsm.Generators[key] - - if !ok { - return - } - - for afterCh := time.After(fifteen * time.Second); ; { - select { - // got a value - case curGenValue, ok := <-g.gchan: - if !ok { - g.Lock() - g.gchan = nil - g.state = done - g.currentGeneratorValue = nil - g.Unlock() - - return - } - - g.currentGeneratorValue = curGenValue - - return - // timeout - case <-afterCh: - g.Lock() - g.gchan = nil - g.state = done - g.Unlock() - - return - } - } -} - -func (gfsm *GeneratorFSM) InitOrSkip(key string) { - gfsm.RLock() - defer gfsm.RUnlock() - - g, ok := gfsm.Generators[key] - if !ok { - return - } - - if len(gfsm.payloads) > 0 { - g.Lock() - defer g.Unlock() - - if g.gchan == nil { - g.gchan = gfsm.generator(gfsm.basePayloads) - g.state = running - } - } -} - -func (gfsm *GeneratorFSM) Value(key string) map[string]interface{} { - gfsm.RLock() - defer gfsm.RUnlock() - - g, ok := gfsm.Generators[key] - if !ok { - return nil - } - - return g.currentGeneratorValue -} - -func (gfsm *GeneratorFSM) Next(key string) bool { - gfsm.RLock() - defer gfsm.RUnlock() - - g, ok := gfsm.Generators[key] - if !ok { - return false - } - - if g.positionPath+g.positionRaw >= len(gfsm.Paths)+len(gfsm.Raws) { - return false - } - - return true -} - -func (gfsm *GeneratorFSM) Position(key string) int { - gfsm.RLock() - defer gfsm.RUnlock() - - g, ok := gfsm.Generators[key] - if !ok { - return 0 - } - - return g.positionPath + g.positionRaw -} - -func (gfsm *GeneratorFSM) Reset(key string) { - gfsm.Lock() - defer gfsm.Unlock() - - if !gfsm.Has(key) { - gfsm.Add(key) - } - - g, ok := gfsm.Generators[key] - if !ok { - return - } - - g.positionPath = 0 - g.positionRaw = 0 -} - -func (gfsm *GeneratorFSM) Current(key string) string { - gfsm.RLock() - defer gfsm.RUnlock() - - g, ok := gfsm.Generators[key] - if !ok { - return "" - } - - if g.positionPath < len(gfsm.Paths) && len(gfsm.Paths) != 0 { - return gfsm.Paths[g.positionPath] - } - - return gfsm.Raws[g.positionRaw] -} -func (gfsm *GeneratorFSM) Total() int { - estimatedRequestsWithPayload := 0 - if len(gfsm.basePayloads) > 0 { - switch gfsm.Type { - case generators.Sniper: - for _, kv := range gfsm.basePayloads { - estimatedRequestsWithPayload += len(kv) - } - case generators.PitchFork: - // Positional so it's equal to the length of one list - for _, kv := range gfsm.basePayloads { - estimatedRequestsWithPayload += len(kv) - break - } - case generators.ClusterBomb: - // Total of combinations => rule of product - prod := 1 - for _, kv := range gfsm.basePayloads { - prod *= len(kv) - } - estimatedRequestsWithPayload += prod - } - } - return len(gfsm.Paths) + len(gfsm.Raws) + estimatedRequestsWithPayload -} - -func (gfsm *GeneratorFSM) Increment(key string) { - gfsm.Lock() - defer gfsm.Unlock() - - g, ok := gfsm.Generators[key] - if !ok { - return - } - - if len(gfsm.Paths) > 0 && g.positionPath < len(gfsm.Paths) { - g.positionPath++ - return - } - - if len(gfsm.Raws) > 0 && g.positionRaw < len(gfsm.Raws) { - // if we have payloads increment only when the generators are done - if g.gchan == nil { - g.state = done - g.positionRaw++ - } - } -} diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go index ce8ba795b..ae4550d18 100644 --- a/v2/pkg/requests/util.go +++ b/v2/pkg/requests/util.go @@ -8,12 +8,6 @@ import ( "strings" ) -const ( - markerParenthesisOpen = "{{" - markerParenthesisClose = "}}" - markerGeneral = "§" -) - func newReplacer(values map[string]interface{}) *strings.Replacer { var replacerItems []string for key, val := range values { @@ -71,7 +65,3 @@ func ExpandMapValues(m map[string]string) (m1 map[string][]string) { } return } - -func hasMarker(s string) bool { - return strings.Contains(s, markerParenthesisOpen) || strings.Contains(s, markerParenthesisClose) || strings.Contains(s, markerGeneral) -} From 651a5edfbba6ababd2f4b1604c77c1b268a6f9c2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 28 Dec 2020 20:02:26 +0530 Subject: [PATCH 31/92] HTTP executor refactor + simplifying logics --- v2/pkg/executer/executer_http.go | 11 - .../protocols/common/generators/generators.go | 18 +- v2/pkg/protocols/http/build_request.go | 19 +- v2/pkg/protocols/http/http.go | 11 + v2/pkg/protocols/http/request.go | 325 ++++++++---------- v2/pkg/requests/doc.go | 3 - v2/pkg/requests/dump.go | 21 -- v2/pkg/requests/util.go | 67 ---- 8 files changed, 177 insertions(+), 298 deletions(-) delete mode 100644 v2/pkg/requests/doc.go delete mode 100644 v2/pkg/requests/dump.go delete mode 100644 v2/pkg/requests/util.go diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 18e86193b..408918d38 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -13,7 +13,6 @@ import ( "regexp" "strconv" "strings" - "sync" "time" "github.com/corpix/uarand" @@ -576,13 +575,3 @@ func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { } } } - -type Result struct { - sync.Mutex - GotResults bool - Meta map[string]interface{} - Matches map[string]interface{} - Extractions map[string]interface{} - historyData map[string]interface{} - Error error -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 4299a2895..b5f099411 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -51,7 +51,8 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) { totalLength = len(v) } } - return &Generator{Type: Type, payloads: compiled}, nil + generator := &Generator{Type: Type, payloads: compiled} + return generator, nil } // Iterator is a single instance of an iterator for a generator structure @@ -88,16 +89,17 @@ func (i *Iterator) Reset() { } } -//Total returns the amount of input combinations available +// Remaining returns the amount of requests left for the generator. +func (i *Iterator) Remaining() int { + return i.total - i.position +} + +// Total returns the amount of input combinations available func (i *Iterator) Total() int { count := 0 switch i.Type { - case Sniper: + case Sniper, PitchFork: count = len(i.payloads[0].values) - case PitchFork: - for _, p := range i.payloads { - count = len(p.values) - } case ClusterBomb: count = 1 for _, p := range i.payloads { @@ -131,6 +133,7 @@ func (i *Iterator) sniperValue() (map[string]interface{}, bool) { } values[payload.name] = payload.value() payload.incrementPosition() + i.position++ return values, true } @@ -145,6 +148,7 @@ func (i *Iterator) pitchforkValue() (map[string]interface{}, bool) { values[p.name] = p.value() p.incrementPosition() } + i.position++ return values, true } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 256031cdb..7357d6686 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -94,10 +95,11 @@ func (r *requestGenerator) nextValue() (string, map[string]interface{}, bool) { // generatedRequest is a single wrapped generated request for a template request type generatedRequest struct { - original *Request - rawRequest *raw.Request - meta map[string]interface{} - request *retryablehttp.Request + original *Request + rawRequest *raw.Request + meta map[string]interface{} + pipelinedClient *rawhttp.PipelineClient + request *retryablehttp.Request } // Make creates a http request for the provided input. @@ -128,6 +130,15 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values) } +// Remaining returns the remaining number of requests for the generator +func (r *requestGenerator) Remaining() int { + if r.payloadIterator != nil { + payloadRemaining := r.payloadIterator.Remaining() + return (len(r.request.Raw) - r.currentIndex + 1) * payloadRemaining + } + return len(r.request.Path) - r.currentIndex + 1 +} + // baseURLWithTemplatePrefs returns the url for BaseURL keeping // the template port and path preference func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index fb40407ee..715ab1a1c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -61,6 +61,7 @@ type Request struct { options *protocols.ExecuterOptions attackType generators.Type + totalRequests int generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client @@ -95,5 +96,15 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } r.options = options + r.totalRequests = r.Requests() return nil } + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + if len(r.Payloads) > 0 { + payloadRequests := r.generator.NewIterator().Total() + return len(r.Raw) * payloadRequests + } + return len(r.Path) +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index e50567502..bbac61073 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -1,128 +1,112 @@ package http -/* -func (e *Request) ExecuteRaceRequest(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strings" + "sync" + "time" - dynamicvalues := make(map[string]interface{}) + "github.com/corpix/uarand" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/requests" + "github.com/projectdiscovery/rawhttp" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/multierr" +) - // verify if the URL is already being processed - if e.HasGenerator(reqURL) { - return result - } +const defaultMaxWorkers = 150 - e.CreateGenerator(reqURL) +// executeRaceRequest executes race condition request for a URL +func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { + generator := e.newGenerator() - // Workers that keeps enqueuing new requests maxWorkers := e.RaceNumberRequests swg := sizedwaitgroup.New(maxWorkers) - for i := 0; i < e.RaceNumberRequests; i++ { - swg.Add() - // base request - result.Lock() - request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) - payloads, _ := e.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") + var requestErr error + var mutex *sync.Mutex + var outputs []*output.InternalWrappedEvent + for i := 0; i < e.RaceNumberRequests; i++ { + request, err := generator.Make(reqURL, nil) + if err != nil { + break + } + + swg.Add() + go func(httpRequest *generatedRequest) { + output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + mutex.Lock() if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") + requestErr = multierr.Append(requestErr, err) + } else { + outputs = append(outputs, output...) } + mutex.Unlock() + swg.Done() }(request) } - swg.Wait() - - return result + return outputs, requestErr } -func (e *Request) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.HasGenerator(reqURL) { - return result - } - - remaining := e.GetRequestCount() - e.CreateGenerator(reqURL) +// executeRaceRequest executes race condition request for a URL +func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { + generator := e.newGenerator() // Workers that keeps enqueuing new requests maxWorkers := e.Threads swg := sizedwaitgroup.New(maxWorkers) - for e.Next(reqURL) { - result.Lock() - request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) - payloads, _ := e.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - e.ratelimiter.Take() - - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - }(request) + var requestErr error + var mutex *sync.Mutex + var outputs []*output.InternalWrappedEvent + for { + request, err := generator.Make(reqURL, dynamicValues) + if err == io.EOF { + break } - p.Update() - e.Increment(reqURL) + if err != nil { + e.options.Progress.DecrementRequests(int64(generator.Remaining())) + return nil, err + } + swg.Add() + go func(httpRequest *generatedRequest) { + defer swg.Done() + + e.options.RateLimiter.Take() + output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + mutex.Lock() + if err != nil { + requestErr = multierr.Append(requestErr, err) + } else { + outputs = append(outputs, output...) + } + mutex.Unlock() + }(request) + e.options.Progress.IncrementRequests() } swg.Wait() - - return result + return outputs, requestErr } -func (e *Request) ExecuteTurboHTTP(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.HasGenerator(reqURL) { - return result - } - - e.CreateGenerator(reqURL) +// executeRaceRequest executes race condition request for a URL +func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { + generator := e.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) if err != nil { - return result + return nil, err } pipeOptions := rawhttp.DefaultPipelineOptions @@ -143,119 +127,90 @@ func (e *Request) ExecuteTurboHTTP(reqURL string) *Result { maxWorkers = pipeOptions.MaxPendingRequests } swg := sizedwaitgroup.New(maxWorkers) - for e.Next(reqURL) { - result.Lock() - request, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) - payloads, _ := e.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - // HTTP pipelining ignores rate limit - // If the request was built correctly then execute it - request.Pipeline = true - request.PipelineClient = pipeclient - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - request.PipelineClient = nil - }(request) + var requestErr error + var mutex *sync.Mutex + var outputs []*output.InternalWrappedEvent + for { + request, err := generator.Make(reqURL, dynamicValues) + if err == io.EOF { + break } + if err != nil { + e.options.Progress.DecrementRequests(int64(generator.Remaining())) + return nil, err + } + request.pipelinedClient = pipeclient - e.Increment(reqURL) + swg.Add() + go func(httpRequest *generatedRequest) { + defer swg.Done() + + output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + mutex.Lock() + if err != nil { + requestErr = multierr.Append(requestErr, err) + } else { + outputs = append(outputs, output...) + } + mutex.Unlock() + }(request) + e.options.Progress.IncrementRequests() } swg.Wait() - return result + return outputs, requestErr } // ExecuteHTTP executes the HTTP request on a URL -func (e *Request) ExecuteHTTP(p *progress.Progress, reqURL string) *Result { +func (e *Request) ExecuteHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // verify if pipeline was requested if e.Pipeline { - return e.ExecuteTurboHTTP(reqURL) + return e.executeTurboHTTP(reqURL, dynamicValues) } // verify if a basic race condition was requested if e.Race && e.RaceNumberRequests > 0 { - return e.ExecuteRaceRequest(reqURL) + return e.executeRaceRequest(reqURL, dynamicValues) } // verify if parallel elaboration was requested if e.Threads > 0 { - return e.ExecuteParallelHTTP(p, reqURL) + return e.executeParallelHTTP(reqURL, dynamicValues) } - var requestNumber int + generator := e.newGenerator() - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - historyData: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.HasGenerator(reqURL) { - return result - } - - remaining := e.GetRequestCount() - e.CreateGenerator(reqURL) - - for e.Next(reqURL) { - requestNumber++ - result.Lock() - httpRequest, err := e.MakeHTTPRequest(reqURL, dynamicvalues, e.Current(reqURL)) - payloads, _ := e.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - e.ratelimiter.Take() - // If the request was built correctly then execute it - format := "%s_" + strconv.Itoa(requestNumber) - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) - if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - e.traceLog.Request(e.template.ID, reqURL, "http", err) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - } - p.Update() - - // Check if has to stop processing at first valid result - if e.stopAtFirstMatch && result.GotResults { - p.Drop(remaining) + var requestErr error + var outputs []*output.InternalWrappedEvent + for { + request, err := generator.Make(reqURL, dynamicValues) + if err == io.EOF { break } + if err != nil { + e.options.Progress.DecrementRequests(int64(generator.Remaining())) + return nil, err + } - // move always forward with requests - e.Increment(reqURL) - remaining-- + e.options.RateLimiter.Take() + output, err := e.executeRequest(reqURL, request, dynamicValues) + if err != nil { + requestErr = multierr.Append(requestErr, err) + } else { + outputs = append(outputs, output...) + } + e.options.Progress.IncrementRequests() + + if request.original.options.Options.StopAtFirstMatch && len(output) > 0 { + e.options.Progress.DecrementRequests(int64(generator.Remaining())) + break + } } - gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) - return result + return outputs, requestErr } -func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { +// executeRequest executes the actual generated request and returns error if occured +func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if e.options.Options.RandomAgent { // nolint:errcheck // ignoring error @@ -285,7 +240,7 @@ func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynam timeStart := time.Now() - if request.Pipeline { + if request.original.Pipeline { resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) if err != nil { if resp != nil { @@ -295,10 +250,10 @@ func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynam return err } e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } else if request.Unsafe { + } else if request.original.Unsafe { // rawhttp // burp uses "\r\n" as new line character - request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") + request.rawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") options := e.rawHTTPClient.Options options.AutomaticContentLength = request.AutomaticContentLengthHeader options.AutomaticHostHeader = request.AutomaticHostHeader @@ -474,6 +429,6 @@ func (e *Request) handleHTTP(reqURL string, request *requests.HTTPRequest, dynam result.Unlock() } + gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) return nil } -*/ diff --git a/v2/pkg/requests/doc.go b/v2/pkg/requests/doc.go deleted file mode 100644 index 3c06053c5..000000000 --- a/v2/pkg/requests/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package requests implements requests for templates that -// will be sent to hosts. -package requests diff --git a/v2/pkg/requests/dump.go b/v2/pkg/requests/dump.go deleted file mode 100644 index 25651513f..000000000 --- a/v2/pkg/requests/dump.go +++ /dev/null @@ -1,21 +0,0 @@ -package requests - -import ( - "bytes" - "io/ioutil" - "net/http/httputil" - "strings" - - "github.com/projectdiscovery/rawhttp" -) - -func Dump(req *HTTPRequest, reqURL string) ([]byte, error) { - if req.Request != nil { - // Create a copy on the fly of the request body - ignore errors - bodyBytes, _ := req.Request.BodyBytes() - req.Request.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) - return httputil.DumpRequest(req.Request.Request, true) - } - - return rawhttp.DumpRequestRaw(req.RawRequest.Method, reqURL, req.RawRequest.Path, ExpandMapValues(req.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.RawRequest.Data))) -} diff --git a/v2/pkg/requests/util.go b/v2/pkg/requests/util.go deleted file mode 100644 index ae4550d18..000000000 --- a/v2/pkg/requests/util.go +++ /dev/null @@ -1,67 +0,0 @@ -package requests - -import ( - "bytes" - "compress/gzip" - "fmt" - "io/ioutil" - "strings" -) - -func newReplacer(values map[string]interface{}) *strings.Replacer { - var replacerItems []string - for key, val := range values { - replacerItems = append( - replacerItems, - fmt.Sprintf("%s%s%s", markerParenthesisOpen, key, markerParenthesisClose), - fmt.Sprintf("%s", val), - fmt.Sprintf("%s%s%s", markerGeneral, key, markerGeneral), - fmt.Sprintf("%s", val), - ) - } - - return strings.NewReplacer(replacerItems...) -} - -// HandleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically) -func HandleDecompression(r *HTTPRequest, bodyOrig []byte) (bodyDec []byte, err error) { - if r.Request == nil { - return bodyOrig, nil - } - - encodingHeader := strings.TrimSpace(strings.ToLower(r.Request.Header.Get("Accept-Encoding"))) - if encodingHeader == "gzip" || encodingHeader == "gzip, deflate" { - gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig)) - if err != nil { - return bodyDec, err - } - defer gzipreader.Close() - - bodyDec, err = ioutil.ReadAll(gzipreader) - if err != nil { - return bodyDec, err - } - - return bodyDec, nil - } - - return bodyOrig, nil -} - -// ZipMapValues converts values from strings slices to flat string -func ZipMapValues(m map[string][]string) (m1 map[string]string) { - m1 = make(map[string]string) - for k, v := range m { - m1[k] = strings.Join(v, "") - } - return -} - -// ExpandMapValues converts values from flat string to strings slice -func ExpandMapValues(m map[string]string) (m1 map[string][]string) { - m1 = make(map[string][]string) - for k, v := range m { - m1[k] = []string{v} - } - return -} From fc8314291744e68501cd62039b0f6aa039fabd2b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 01:30:07 +0530 Subject: [PATCH 32/92] Misc work on making http protocol runnable --- v2/pkg/operators/operators.go | 2 + v2/pkg/protocols/common/generators/maps.go | 9 + v2/pkg/protocols/http/executer.go | 90 ++++++++ v2/pkg/protocols/http/http.go | 2 + v2/pkg/protocols/http/operators.go | 38 +++- v2/pkg/protocols/http/request.go | 253 ++++++++------------- v2/pkg/protocols/http/utils.go | 75 ++++++ v2/pkg/protocols/protocols.go | 3 + 8 files changed, 319 insertions(+), 153 deletions(-) create mode 100644 v2/pkg/protocols/http/executer.go create mode 100644 v2/pkg/protocols/http/utils.go diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 960509045..16c2e8f4b 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -57,6 +57,8 @@ type Result struct { OutputExtracts []string // DynamicValues contains any dynamic values to be templated DynamicValues map[string]string + // PayloadValues contains payload values provided by user. (Optional) + PayloadValues map[string]interface{} } // MatchFunc performs matching operation for a matcher on model and returns true or false. diff --git a/v2/pkg/protocols/common/generators/maps.go b/v2/pkg/protocols/common/generators/maps.go index da78af9d3..75779f2b7 100644 --- a/v2/pkg/protocols/common/generators/maps.go +++ b/v2/pkg/protocols/common/generators/maps.go @@ -14,6 +14,15 @@ func MergeMaps(m1, m2 map[string]interface{}) map[string]interface{} { return m } +// ExpandMapValues converts values from flat string to strings slice +func ExpandMapValues(m map[string]string) map[string][]string { + m1 := make(map[string][]string, len(m)) + for k, v := range m { + m1[k] = []string{v} + } + return m1 +} + // CopyMap creates a new copy of an existing map func CopyMap(originalMap map[string]interface{}) map[string]interface{} { newMap := make(map[string]interface{}) diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go new file mode 100644 index 000000000..fa4031302 --- /dev/null +++ b/v2/pkg/protocols/http/executer.go @@ -0,0 +1,90 @@ +package http + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Executer executes a group of requests for a protocol +type Executer struct { + requests []*Request + options *protocols.ExecuterOptions +} + +var _ protocols.Executer = &Executer{} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { + return &Executer{requests: requests, options: options} +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + for _, request := range e.requests { + err := request.Compile(e.options) + if err != nil { + return err + } + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int64 { + var count int64 + for _, request := range e.requests { + count += int64(request.Requests()) + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + for _, req := range e.requests { + events, err := req.ExecuteHTTP(input, nil) + if err != nil { + return false, err + } + if events == nil { + return false, nil + } + + // If we have a result field, we should add a result to slice. + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) + } + } + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + var results []*output.InternalWrappedEvent + + for _, req := range e.requests { + events, err := req.ExecuteHTTP(input, nil) + if err != nil { + return nil, err + } + if events == nil { + return nil, nil + } + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + event.Results = req.makeResultEvent(event) + } + results = append(results, events...) + } + return results, nil +} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 715ab1a1c..949a210ba 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -62,6 +63,7 @@ type Request struct { options *protocols.ExecuterOptions attackType generators.Type totalRequests int + customHeaders types.StringSlice generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index c68cf036f..2e15eddde 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -84,12 +85,17 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a HTTP response to a map for use in DSL matching -func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { +func (r *Request) responseToDSLMap(resp *http.Response, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } + if r.options.Options.JSONRequests { + data["request"] = rawReq + data["response"] = rawResp + } + data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode @@ -110,3 +116,33 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D data["duration"] = duration.Seconds() return data } + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) + + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "http", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Request = wrapped.InternalEvent["request"].(string) + data.Response = wrapped.InternalEvent["raw"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index bbac61073..28c061d0b 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -15,10 +15,9 @@ import ( "github.com/corpix/uarand" "github.com/pkg/errors" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/rawhttp" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" @@ -213,10 +212,12 @@ func (e *Request) ExecuteHTTP(reqURL string, dynamicValues map[string]interface{ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if e.options.Options.RandomAgent { + builder := &strings.Builder{} + builder.WriteString("User-Agent: ") // nolint:errcheck // ignoring error - e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) + builder.WriteString(uarand.GetRandom()) + e.customHeaders.Set(builder.String()) } - e.setCustomHeaders(request) var ( @@ -225,210 +226,158 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam dumpedRequest []byte fromcache bool ) - - if e.debug || e.pf != nil { - dumpedRequest, err = requests.Dump(request, reqURL) + if e.options.Options.Debug || e.options.ProjectFile != nil { + dumpedRequest, err = dump(request, reqURL) if err != nil { - return err + return nil, err } } - - if e.debug { - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID) + if e.options.Options.Debug { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", e.options.TemplateID, reqURL) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } timeStart := time.Now() - if request.original.Pipeline { - resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) + resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) } else if request.original.Unsafe { // rawhttp // burp uses "\r\n" as new line character - request.rawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") - options := e.rawHTTPClient.Options - options.AutomaticContentLength = request.AutomaticContentLengthHeader - options.AutomaticHostHeader = request.AutomaticHostHeader - options.FollowRedirects = request.FollowRedirects - resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) + request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n") + options := request.original.rawhttpClient.Options + options.AutomaticContentLength = !e.DisableAutoContentLength + options.AutomaticHostHeader = !e.DisableAutoHostname + options.FollowRedirects = e.Redirects + resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) } else { // if nuclei-project is available check if the request was already sent previously - if e.pf != nil { + if e.options.ProjectFile != nil { // if unavailable fail silently fromcache = true // nolint:bodyclose // false positive the response is generated at runtime - resp, err = e.pf.Get(dumpedRequest) + resp, err = e.options.ProjectFile.Get(dumpedRequest) if err != nil { fromcache = false } } - - // retryablehttp if resp == nil { - resp, err = e.httpClient.Do(request.Request) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) + resp, err = e.httpClient.Do(request.request) } } + if err != nil { + if resp != nil { + _, _ = io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } + e.options.Output.Request(e.options.TemplateID, reqURL, "http", err) + e.options.Progress.DecrementRequests(1) + return nil, err + } + e.options.Output.Request(e.options.TemplateID, reqURL, "http", err) duration := time.Since(timeStart) - // Dump response - Step 1 - Decompression not yet handled var dumpedResponse []byte - if e.debug { + if e.options.Options.Debug { var dumpErr error dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { - return errors.Wrap(dumpErr, "could not dump http response") + return nil, errors.Wrap(dumpErr, "could not dump http response") } } data, err := ioutil.ReadAll(resp.Body) if err != nil { - _, copyErr := io.Copy(ioutil.Discard, resp.Body) - if copyErr != nil { - resp.Body.Close() - return copyErr - } - + _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - - return errors.Wrap(err, "could not read http body") + return nil, errors.Wrap(err, "could not read http body") } - resp.Body.Close() - // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request - // so in case we have to manually do it + // net/http doesn't automatically decompress the response body if an + // encoding has been specified by the user in the request so in case we have to + // manually do it. dataOrig := data - data, err = requests.HandleDecompression(request, data) + data, err = handleDecompression(request, data) if err != nil { - return errors.Wrap(err, "could not decompress http body") + return nil, errors.Wrap(err, "could not decompress http body") } // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if e.debug { + if e.options.Options.Debug { dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", e.options.TemplateID, reqURL) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) } // if nuclei-project is enabled store the response if not previously done - if e.pf != nil && !fromcache { - err := e.pf.Set(dumpedRequest, resp, data) + if e.options.ProjectFile != nil && !fromcache { + err := e.options.ProjectFile.Set(dumpedRequest, resp, data) if err != nil { - return errors.Wrap(err, "could not store in project file") + return nil, errors.Wrap(err, "could not store in project file") } } - // Convert response body from []byte to string with zero copy - body := unsafeToString(data) - - headers := headersToString(resp.Header) - - var matchData map[string]interface{} - if payloads != nil { - matchData = generators.MergeMaps(result.historyData, payloads) - } + // var matchData map[string]interface{} + // if payloads != nil { + // matchData = generators.MergeMaps(result.historyData, payloads) + // } // store for internal purposes the DSL matcher data // hardcode stopping storing data after defaultMaxHistorydata items - if len(result.historyData) < defaultMaxHistorydata { - result.Lock() - // update history data with current reqURL and hostname - result.historyData["reqURL"] = reqURL - if parsed, err := url.Parse(reqURL); err == nil { - result.historyData["Hostname"] = parsed.Host + //if len(result.historyData) < defaultMaxHistorydata { + // result.Lock() + // // update history data with current reqURL and hostname + // result.historyData["reqURL"] = reqURL + // if parsed, err := url.Parse(reqURL); err == nil { + // result.historyData["Hostname"] = parsed.Host + // } + // result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) + // if payloads == nil { + // // merge them to history data + // result.historyData = generators.MergeMaps(result.historyData, payloads) + // } + // result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) + // + // // complement match data with new one if necessary + // matchData = generators.MergeMaps(matchData, result.historyData) + // result.Unlock() + //} + ouputEvent := e.responseToDSLMap(resp, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) + + event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + if e.Operators != nil { + result, ok := e.Operators.Execute(ouputEvent, e.Match, e.Extract) + if !ok { + return nil, nil + } + result.PayloadValues = request.meta + event[0].OperatorsResult = result + } + return event, nil +} + +const two = 2 + +// setCustomHeaders sets the custom headers for generated request +func (e *Request) setCustomHeaders(r *generatedRequest) { + for _, customHeader := range e.customHeaders { + if customHeader == "" { + continue + } + + // This should be pre-computed somewhere and done only once + tokens := strings.SplitN(customHeader, ":", two) + // if it's an invalid header skip it + if len(tokens) < 2 { + continue + } + + headerName, headerValue := tokens[0], strings.Join(tokens[1:], "") + if r.rawRequest != nil { + r.rawRequest.Headers[headerName] = headerValue + } else { + r.request.Header.Set(strings.TrimSpace(headerName), strings.TrimSpace(headerValue)) } - result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) - if payloads == nil { - // merge them to history data - result.historyData = generators.MergeMaps(result.historyData, payloads) - } - result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) - - // complement match data with new one if necessary - matchData = generators.MergeMaps(matchData, result.historyData) - result.Unlock() } - - matcherCondition := e.GetMatchersCondition() - for _, matcher := range e.Matchers { - // Check if the matcher matched - if !matcher.Match(resp, body, headers, duration, matchData) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - return nil - } - } else { - // If the matcher has matched, and its an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition { - result.Lock() - result.Matches[matcher.Name] = nil - // probably redundant but ensures we snapshot current payload values when matchers are valid - result.Meta = request.Meta - result.GotResults = true - result.Unlock() - e.writeOutputHTTP(request, resp, body, matcher, nil, request.Meta, reqURL) - } - } - } - - // All matchers have successfully completed so now start with the - // next task which is extraction of input from matchers. - var extractorResults, outputExtractorResults []string - - for _, extractor := range e.Extractors { - for match := range extractor.Extract(resp, body, headers) { - if _, ok := dynamicvalues[extractor.Name]; !ok { - dynamicvalues[extractor.Name] = match - } - - extractorResults = append(extractorResults, match) - - if !extractor.Internal { - outputExtractorResults = append(outputExtractorResults, match) - } - } - // probably redundant but ensures we snapshot current payload values when extractors are valid - result.Lock() - result.Meta = request.Meta - result.Extractions[extractor.Name] = extractorResults - result.Unlock() - } - - // Write a final string of output if matcher type is - // AND or if we have extractors for the mechanism too. - if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults, request.Meta, reqURL) - result.Lock() - result.GotResults = true - result.Unlock() - } - - gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) - return nil } diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go new file mode 100644 index 000000000..5629307c9 --- /dev/null +++ b/v2/pkg/protocols/http/utils.go @@ -0,0 +1,75 @@ +package http + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + "net/http" + "net/http/httputil" + "strings" + "unsafe" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/rawhttp" +) + +// unsafeToString converts byte slice to string with zero allocations +func unsafeToString(bs []byte) string { + return *(*string)(unsafe.Pointer(&bs)) +} + +// headersToString converts http headers to string +func headersToString(headers http.Header) string { + builder := &strings.Builder{} + + for header, values := range headers { + builder.WriteString(header) + builder.WriteString(": ") + + for i, value := range values { + builder.WriteString(value) + + if i != len(values)-1 { + builder.WriteRune('\n') + builder.WriteString(header) + builder.WriteString(": ") + } + } + builder.WriteRune('\n') + } + return builder.String() +} + +// dump creates a dump of the http request in form of a byte slice +func dump(req *generatedRequest, reqURL string) ([]byte, error) { + if req.request != nil { + // Create a copy on the fly of the request body - ignore errors + bodyBytes, _ := req.request.BodyBytes() + req.request.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + return httputil.DumpRequest(req.request.Request, true) + } + return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(req.rawRequest.Data))) +} + +// handleDecompression if the user specified a custom encoding (as golang transport doesn't do this automatically) +func handleDecompression(r *generatedRequest, bodyOrig []byte) (bodyDec []byte, err error) { + if r.request == nil { + return bodyOrig, nil + } + + encodingHeader := strings.TrimSpace(strings.ToLower(r.request.Header.Get("Accept-Encoding"))) + if encodingHeader == "gzip" || encodingHeader == "gzip, deflate" { + gzipreader, err := gzip.NewReader(bytes.NewReader(bodyOrig)) + if err != nil { + return bodyDec, err + } + defer gzipreader.Close() + + bodyDec, err = ioutil.ReadAll(gzipreader) + if err != nil { + return bodyDec, err + } + return bodyDec, nil + } + return bodyOrig, nil +} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 6aad81e73..9be2ef1c1 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" ) @@ -35,6 +36,8 @@ type ExecuterOptions struct { Progress *progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. RateLimiter ratelimit.Limiter + // ProjectFile is the project file for nuclei + ProjectFile *projectfile.ProjectFile } // Request is an interface implemented any protocol based request generator. From 97ad8e592ec25a9896d891fb2c881e545b18cdc4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 11:42:46 +0530 Subject: [PATCH 33/92] Working DNS and HTTP protocol implm --- v2/pkg/output/output.go | 2 + .../protocols/dns/dnsclientpool/clientpool.go | 8 +- v2/pkg/protocols/dns/executer_test.go | 73 +++++++++++++++++++ v2/pkg/protocols/dns/operators.go | 11 +-- v2/pkg/protocols/http/build_request.go | 10 +-- v2/pkg/protocols/http/executer.go | 4 +- v2/pkg/protocols/http/executer_test.go | 47 ++++++++++++ .../http/httpclientpool/clientpool.go | 11 ++- v2/pkg/protocols/http/operators.go | 22 ++---- v2/pkg/protocols/http/request.go | 13 +++- 10 files changed, 157 insertions(+), 44 deletions(-) create mode 100644 v2/pkg/protocols/dns/executer_test.go create mode 100644 v2/pkg/protocols/http/executer_test.go diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 25085887e..143bccab1 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -130,6 +130,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { return errors.Wrap(err, "could not format output") } _, _ = os.Stdout.Write(data) + _, _ = os.Stdout.Write([]byte("\n")) if w.outputFile != nil { if !w.json { data = decolorizerRegex.ReplaceAll(data, []byte("")) @@ -137,6 +138,7 @@ func (w *StandardWriter) Write(event *ResultEvent) error { if writeErr := w.outputFile.Write(data); writeErr != nil { return errors.Wrap(err, "could not write to output") } + _ = w.outputFile.Write([]byte("\n")) } return nil } diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 8b05f093e..23411469a 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -32,11 +32,7 @@ func Init(options *types.Options) error { poolMutex = &sync.RWMutex{} clientPool = make(map[string]*retryabledns.Client) - if client, err := Get(options, &Configuration{}); err != nil { - return err - } else { - normalClient = client - } + normalClient = retryabledns.New(defaultResolvers, 1) return nil } @@ -58,7 +54,7 @@ func (c *Configuration) Hash() string { // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { - if !(configuration.Retries > 0) { + if !(configuration.Retries > 1) { return normalClient, nil } hash := configuration.Hash() diff --git a/v2/pkg/protocols/dns/executer_test.go b/v2/pkg/protocols/dns/executer_test.go new file mode 100644 index 000000000..9143eabaf --- /dev/null +++ b/v2/pkg/protocols/dns/executer_test.go @@ -0,0 +1,73 @@ +package dns + +import ( + "fmt" + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestRequest(t *testing.T) { + err := dnsclientpool.Init(&types.Options{}) + require.Nil(t, err, "could not initialize dns client pool") + + writer, err := output.NewStandardWriter(true, false, false, "", "") + require.Nil(t, err, "could not create standard output writer") + + progress, err := progress.NewProgress(false, false, 0) + require.Nil(t, err, "could not create standard progress writer") + + protocolOpts := &protocols.ExecuterOptions{ + TemplateID: "testing-dns", + TemplateInfo: map[string]string{"author": "test"}, + Output: writer, + Options: &types.Options{}, + Progress: progress, + } + req := &Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ + Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, + }} + err = req.Compile(protocolOpts) + require.Nil(t, err, "could not compile request") + + output, err := req.ExecuteWithResults("docs.hackerone.com.", nil) + require.Nil(t, err, "could not execute request") + + for _, result := range output { + fmt.Printf("%+v\n", result) + } +} + +func TestExecuter(t *testing.T) { + err := dnsclientpool.Init(&types.Options{}) + require.Nil(t, err, "could not initialize dns client pool") + + writer, err := output.NewStandardWriter(true, false, false, "", "") + require.Nil(t, err, "could not create standard output writer") + + progress, err := progress.NewProgress(false, false, 0) + require.Nil(t, err, "could not create standard progress writer") + + protocolOpts := &protocols.ExecuterOptions{ + TemplateID: "testing-dns", + TemplateInfo: map[string]string{"author": "test"}, + Output: writer, + Options: &types.Options{}, + Progress: progress, + } + executer := NewExecuter([]*Request{&Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ + Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, + }}}, protocolOpts) + err = executer.Compile() + require.Nil(t, err, "could not compile request") + + _, err = executer.Execute("docs.hackerone.com") + require.Nil(t, err, "could not execute request") +} diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 5ea850a9b..88c1aed2e 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -12,14 +12,9 @@ import ( // Match matches a generic data response again a given matcher func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { - part, ok := data[matcher.Part] - if !ok { - return false - } - partString := part.(string) - + partString := matcher.Part switch partString { - case "body", "all": + case "body", "all", "": partString = "raw" } @@ -123,7 +118,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out // makeResultEvent creates a result event from internal wrapped event func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) data := output.ResultEvent{ TemplateID: r.options.TemplateID, diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 7357d6686..49794c52b 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -168,7 +168,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st if err != nil { return nil, err } - return &generatedRequest{request: request}, nil + return &generatedRequest{request: request, original: r.request}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request @@ -221,11 +221,7 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, // rawhttp if r.request.Unsafe { - unsafeReq := &generatedRequest{ - rawRequest: rawRequestData, - meta: genValues, - original: r.request, - } + unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: genValues, original: r.request} return unsafeReq, nil } @@ -252,7 +248,7 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, if err != nil { return nil, err } - return &generatedRequest{request: request, meta: genValues}, nil + return &generatedRequest{request: request, meta: genValues, original: r.request}, nil } // fillRequest fills various headers in the request with values diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go index fa4031302..c4c8724c7 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/http/executer.go @@ -43,7 +43,7 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool for _, req := range e.requests { - events, err := req.ExecuteHTTP(input, nil) + events, err := req.ExecuteWithResults(input, nil) if err != nil { return false, err } @@ -71,7 +71,7 @@ func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEv var results []*output.InternalWrappedEvent for _, req := range e.requests { - events, err := req.ExecuteHTTP(input, nil) + events, err := req.ExecuteWithResults(input, nil) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/http/executer_test.go b/v2/pkg/protocols/http/executer_test.go new file mode 100644 index 000000000..a8bafdb26 --- /dev/null +++ b/v2/pkg/protocols/http/executer_test.go @@ -0,0 +1,47 @@ +package http + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" + "go.uber.org/ratelimit" +) + +func TestRequest(t *testing.T) { + err := httpclientpool.Init(&types.Options{}) + require.Nil(t, err, "could not initialize dns client pool") + + writer, err := output.NewStandardWriter(true, false, false, "", "") + require.Nil(t, err, "could not create standard output writer") + + progress, err := progress.NewProgress(false, false, 0) + require.Nil(t, err, "could not create standard progress writer") + + protocolOpts := &protocols.ExecuterOptions{ + TemplateID: "testing-dns", + TemplateInfo: map[string]string{"author": "test"}, + Output: writer, + Options: &types.Options{}, + Progress: progress, + RateLimiter: ratelimit.New(100), + } + executer := NewExecuter([]*Request{&Request{Path: []string{"{{BaseURL}}"}, Method: "GET", Operators: &operators.Operators{ + Matchers: []*matchers.Matcher{{Type: "dsl", DSL: []string{"!contains(tolower(all_headers), 'x-frame-options')"}, Part: "body"}}, + }}}, protocolOpts) + err = executer.Compile() + require.Nil(t, err, "could not compile request") + + _, err = executer.Execute("https://example.com") + require.Nil(t, err, "could not execute request") + + // for _, result := range output { + // fmt.Printf("%+v\n", result) + // } +} diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index d0f94d223..4ae10f87a 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -37,11 +37,11 @@ func Init(options *types.Options) error { poolMutex = &sync.RWMutex{} clientPool = make(map[string]*retryablehttp.Client) - if client, err := Get(options, &Configuration{}); err != nil { + client, err := wrappedGet(options, &Configuration{}) + if err != nil { return err - } else { - normalClient = client } + normalClient = client return nil } @@ -82,6 +82,11 @@ func Get(options *types.Options, configuration *Configuration) (*retryablehttp.C if !(configuration.Threads > 0 && configuration.MaxRedirects > 0 && configuration.FollowRedirects) { return normalClient, nil } + return wrappedGet(options, configuration) +} + +// wrappedGet wraps a get operation without normal cliet check +func wrappedGet(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { var proxyURL *url.URL var err error diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 2e15eddde..a3e15d898 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -14,12 +14,7 @@ import ( // Match matches a generic data response again a given matcher func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { - part, ok := data[matcher.Part] - if !ok { - return false - } - partString := part.(string) - + partString := matcher.Part switch partString { case "header": partString = "all_headers" @@ -56,12 +51,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) // Extract performs extracting operation for a extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - part, ok := data[extractor.Part] - if !ok { - return nil - } - partString := part.(string) - + partString := extractor.Part switch partString { case "header": partString = "all_headers" @@ -85,12 +75,14 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies())) +func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { + data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } + data["host"] = host + data["matched"] = matched if r.options.Options.JSONRequests { data["request"] = rawReq data["response"] = rawResp @@ -119,7 +111,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, rawReq, rawResp, body, h // makeResultEvent creates a result event from internal wrapped event func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - results := make([]*output.ResultEvent, len(wrapped.OperatorsResult.Matches)+1) + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) data := output.ResultEvent{ TemplateID: r.options.TemplateID, diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 28c061d0b..89dcb74da 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -160,8 +160,8 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter return outputs, requestErr } -// ExecuteHTTP executes the HTTP request on a URL -func (e *Request) ExecuteHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +// ExecuteWithResults executes the final request on a URL +func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // verify if pipeline was requested if e.Pipeline { return e.executeTurboHTTP(reqURL, dynamicValues) @@ -343,7 +343,14 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam // matchData = generators.MergeMaps(matchData, result.historyData) // result.Unlock() //} - ouputEvent := e.responseToDSLMap(resp, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) + var matchedURL string + if request.rawRequest != nil { + matchedURL = request.rawRequest.FullURL + } + if request.request != nil { + matchedURL = request.request.URL.String() + } + ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} if e.Operators != nil { From aefa2717f7d6d1702e3008ac18ebdcc3660dc256 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 12:08:46 +0530 Subject: [PATCH 34/92] Added payload validation + misc --- v2/pkg/executer/executer_http.go | 577 ------------------ v2/pkg/executer/output_http.go | 147 ----- v2/pkg/executer/utils.go | 59 -- .../protocols/common/generators/generators.go | 10 +- .../common/generators/generators_test.go | 6 +- .../protocols/common/generators/validate.go | 61 ++ v2/pkg/protocols/dns/executer_test.go | 73 --- v2/pkg/protocols/http/build_request_test.go | 4 +- v2/pkg/protocols/http/executer_test.go | 47 -- v2/pkg/protocols/http/http.go | 9 +- v2/pkg/protocols/protocols.go | 2 + v2/pkg/templates/compile.go | 94 --- 12 files changed, 83 insertions(+), 1006 deletions(-) delete mode 100644 v2/pkg/executer/executer_http.go delete mode 100644 v2/pkg/executer/output_http.go delete mode 100644 v2/pkg/executer/utils.go create mode 100644 v2/pkg/protocols/common/generators/validate.go delete mode 100644 v2/pkg/protocols/dns/executer_test.go delete mode 100644 v2/pkg/protocols/http/executer_test.go diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go deleted file mode 100644 index 408918d38..000000000 --- a/v2/pkg/executer/executer_http.go +++ /dev/null @@ -1,577 +0,0 @@ -package executer - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/cookiejar" - "net/http/httputil" - "net/url" - "os" - "regexp" - "strconv" - "strings" - "time" - - "github.com/corpix/uarand" - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/internal/tracelog" - "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/rawhttp" - "github.com/projectdiscovery/retryablehttp-go" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/ratelimit" -) - -const ( - two = 2 - ten = 10 - defaultMaxWorkers = 150 - defaultMaxHistorydata = 150 -) - -// HTTPExecuter is client for performing HTTP requests -// for a template. -type HTTPExecuter struct { - pf *projetctfile.ProjectFile - customHeaders requests.CustomHeaders - colorizer colorizer.NucleiColorizer - httpClient *retryablehttp.Client - rawHTTPClient *rawhttp.Client - template *templates.Template - bulkHTTPRequest *requests.BulkHTTPRequest - writer *bufwriter.Writer - CookieJar *cookiejar.Jar - traceLog tracelog.Log - decolorizer *regexp.Regexp - randomAgent bool - vhost bool - coloredOutput bool - debug bool - Results bool - jsonOutput bool - jsonRequest bool - noMeta bool - stopAtFirstMatch bool - ratelimiter ratelimit.Limiter -} - -// HTTPOptions contains configuration options for the HTTP executer. -type HTTPOptions struct { - Template *templates.Template - BulkHTTPRequest *requests.BulkHTTPRequest - CookieJar *cookiejar.Jar - PF *projetctfile.ProjectFile -} - -// NewHTTPExecuter creates a new HTTP executer from a template -// and a HTTP request query. -func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { - var ( - proxyURL *url.URL - err error - ) - - if err != nil { - return nil, err - } - - // Create the HTTP Client - client := makeHTTPClient(proxyURL, options) - // nolint:bodyclose // false positive there is no body to close yet - - if options.CookieJar != nil { - client.HTTPClient.Jar = options.CookieJar - } else if options.CookieReuse { - jar, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - client.HTTPClient.Jar = jar - } - - executer := &HTTPExecuter{ - debug: options.Debug, - jsonOutput: options.JSON, - jsonRequest: options.JSONRequests, - noMeta: options.NoMeta, - httpClient: client, - rawHTTPClient: rawClient, - traceLog: options.TraceLog, - template: options.Template, - bulkHTTPRequest: options.BulkHTTPRequest, - writer: options.Writer, - randomAgent: options.RandomAgent, - customHeaders: options.CustomHeaders, - CookieJar: options.CookieJar, - coloredOutput: options.ColoredOutput, - colorizer: *options.Colorizer, - decolorizer: options.Decolorizer, - stopAtFirstMatch: options.StopAtFirstMatch, - pf: options.PF, - vhost: options.Vhost, - ratelimiter: options.RateLimiter, - } - return executer, nil -} - -func (e *HTTPExecuter) ExecuteRaceRequest(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // Workers that keeps enqueuing new requests - maxWorkers := e.bulkHTTPRequest.RaceNumberRequests - swg := sizedwaitgroup.New(maxWorkers) - for i := 0; i < e.bulkHTTPRequest.RaceNumberRequests; i++ { - swg.Add() - // base request - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") - } - }(request) - } - - swg.Wait() - - return result -} - -func (e *HTTPExecuter) ExecuteParallelHTTP(p *progress.Progress, reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - remaining := e.bulkHTTPRequest.GetRequestCount() - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // Workers that keeps enqueuing new requests - maxWorkers := e.bulkHTTPRequest.Threads - swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) { - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - e.ratelimiter.Take() - - // If the request was built correctly then execute it - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - }(request) - } - p.Update() - e.bulkHTTPRequest.Increment(reqURL) - } - swg.Wait() - - return result -} - -func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result { - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - e.bulkHTTPRequest.CreateGenerator(reqURL) - - // need to extract the target from the url - URL, err := url.Parse(reqURL) - if err != nil { - return result - } - - // defaultMaxWorkers should be a sufficient value to keep queues always full - maxWorkers := defaultMaxWorkers - // in case the queue is bigger increase the workers - if pipeOptions.MaxPendingRequests > maxWorkers { - maxWorkers = pipeOptions.MaxPendingRequests - } - swg := sizedwaitgroup.New(maxWorkers) - for e.bulkHTTPRequest.Next(reqURL) { - result.Lock() - request, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - } else { - swg.Add() - go func(httpRequest *requests.HTTPRequest) { - defer swg.Done() - - // HTTP pipelining ignores rate limit - // If the request was built correctly then execute it - request.Pipeline = true - request.PipelineClient = pipeclient - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, "") - if err != nil { - e.traceLog.Request(e.template.ID, reqURL, "http", err) - result.Error = errors.Wrap(err, "could not handle http request") - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - request.PipelineClient = nil - }(request) - } - - e.bulkHTTPRequest.Increment(reqURL) - } - swg.Wait() - return result -} - -// ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, reqURL string) *Result { - var customHost string - if e.vhost { - parts := strings.Split(reqURL, ",") - reqURL = parts[0] - customHost = parts[1] - } - - // verify if pipeline was requested - if e.bulkHTTPRequest.Pipeline { - return e.ExecuteTurboHTTP(reqURL) - } - - // verify if a basic race condition was requested - if e.bulkHTTPRequest.Race && e.bulkHTTPRequest.RaceNumberRequests > 0 { - return e.ExecuteRaceRequest(reqURL) - } - - // verify if parallel elaboration was requested - if e.bulkHTTPRequest.Threads > 0 { - return e.ExecuteParallelHTTP(p, reqURL) - } - - var requestNumber int - - result := &Result{ - Matches: make(map[string]interface{}), - Extractions: make(map[string]interface{}), - historyData: make(map[string]interface{}), - } - - dynamicvalues := make(map[string]interface{}) - - // verify if the URL is already being processed - if e.bulkHTTPRequest.HasGenerator(reqURL) { - return result - } - - remaining := e.bulkHTTPRequest.GetRequestCount() - e.bulkHTTPRequest.CreateGenerator(reqURL) - - for e.bulkHTTPRequest.Next(reqURL) { - requestNumber++ - result.Lock() - httpRequest, err := e.bulkHTTPRequest.MakeHTTPRequest(reqURL, dynamicvalues, e.bulkHTTPRequest.Current(reqURL)) - payloads, _ := e.bulkHTTPRequest.GetPayloadsValues(reqURL) - result.Unlock() - // ignore the error due to the base request having null paylods - if err == requests.ErrNoPayload { - // pass through - } else if err != nil { - result.Error = err - p.Drop(remaining) - } else { - if e.vhost { - if httpRequest.Request != nil { - httpRequest.Request.Host = customHost - } - if httpRequest.RawRequest != nil && httpRequest.RawRequest.Headers != nil { - httpRequest.RawRequest.Headers["Host"] = customHost - } - } - - e.ratelimiter.Take() - // If the request was built correctly then execute it - format := "%s_" + strconv.Itoa(requestNumber) - err = e.handleHTTP(reqURL, httpRequest, dynamicvalues, result, payloads, format) - if err != nil { - result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) - e.traceLog.Request(e.template.ID, reqURL, "http", err) - } else { - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - } - p.Update() - - // Check if has to stop processing at first valid result - if e.stopAtFirstMatch && result.GotResults { - p.Drop(remaining) - break - } - - // move always forward with requests - e.bulkHTTPRequest.Increment(reqURL) - remaining-- - } - gologger.Verbosef("Sent for [%s] to %s\n", "http-request", e.template.ID, reqURL) - return result -} - -func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest, dynamicvalues map[string]interface{}, result *Result, payloads map[string]interface{}, format string) error { - // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if e.randomAgent { - // nolint:errcheck // ignoring error - e.customHeaders.Set("User-Agent: " + uarand.GetRandom()) - } - - e.setCustomHeaders(request) - - var ( - resp *http.Response - err error - dumpedRequest []byte - fromcache bool - ) - - if e.debug || e.pf != nil { - dumpedRequest, err = requests.Dump(request, reqURL) - if err != nil { - return err - } - } - - if e.debug { - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) - } - - timeStart := time.Now() - - if request.Pipeline { - resp, err = request.PipelineClient.DoRaw(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data))) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } else if request.Unsafe { - // rawhttp - // burp uses "\r\n" as new line character - request.RawRequest.Data = strings.ReplaceAll(request.RawRequest.Data, "\n", "\r\n") - options := e.rawHTTPClient.Options - options.AutomaticContentLength = request.AutomaticContentLengthHeader - options.AutomaticHostHeader = request.AutomaticHostHeader - options.FollowRedirects = request.FollowRedirects - resp, err = e.rawHTTPClient.DoRawWithOptions(request.RawRequest.Method, reqURL, request.RawRequest.Path, requests.ExpandMapValues(request.RawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.RawRequest.Data)), options) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } else { - // if nuclei-project is available check if the request was already sent previously - if e.pf != nil { - // if unavailable fail silently - fromcache = true - // nolint:bodyclose // false positive the response is generated at runtime - resp, err = e.pf.Get(dumpedRequest) - if err != nil { - fromcache = false - } - } - - // retryablehttp - if resp == nil { - resp, err = e.httpClient.Do(request.Request) - if err != nil { - if resp != nil { - resp.Body.Close() - } - e.traceLog.Request(e.template.ID, reqURL, "http", err) - return err - } - e.traceLog.Request(e.template.ID, reqURL, "http", nil) - } - } - - duration := time.Since(timeStart) - - // Dump response - Step 1 - Decompression not yet handled - var dumpedResponse []byte - if e.debug { - var dumpErr error - dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) - if dumpErr != nil { - return errors.Wrap(dumpErr, "could not dump http response") - } - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - _, copyErr := io.Copy(ioutil.Discard, resp.Body) - if copyErr != nil { - resp.Body.Close() - return copyErr - } - - resp.Body.Close() - - return errors.Wrap(err, "could not read http body") - } - - resp.Body.Close() - - // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request - // so in case we have to manually do it - dataOrig := data - data, err = requests.HandleDecompression(request, data) - if err != nil { - return errors.Wrap(err, "could not decompress http body") - } - - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if e.debug { - dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", reqURL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) - } - - // if nuclei-project is enabled store the response if not previously done - if e.pf != nil && !fromcache { - err := e.pf.Set(dumpedRequest, resp, data) - if err != nil { - return errors.Wrap(err, "could not store in project file") - } - } - - // Convert response body from []byte to string with zero copy - body := unsafeToString(data) - - headers := headersToString(resp.Header) - - var matchData map[string]interface{} - if payloads != nil { - matchData = generators.MergeMaps(result.historyData, payloads) - } - - // store for internal purposes the DSL matcher data - // hardcode stopping storing data after defaultMaxHistorydata items - if len(result.historyData) < defaultMaxHistorydata { - result.Lock() - // update history data with current reqURL and hostname - result.historyData["reqURL"] = reqURL - if parsed, err := url.Parse(reqURL); err == nil { - result.historyData["Hostname"] = parsed.Host - } - result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) - if payloads == nil { - // merge them to history data - result.historyData = generators.MergeMaps(result.historyData, payloads) - } - result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) - - // complement match data with new one if necessary - matchData = generators.MergeMaps(matchData, result.historyData) - result.Unlock() - } - - return nil -} - -// Close closes the http executer for a template. -func (e *HTTPExecuter) Close() {} - -func (e *HTTPExecuter) setCustomHeaders(r *requests.HTTPRequest) { - for _, customHeader := range e.customHeaders { - // This should be pre-computed somewhere and done only once - tokens := strings.SplitN(customHeader, ":", two) - // if it's an invalid header skip it - if len(tokens) < two { - continue - } - - headerName, headerValue := tokens[0], strings.Join(tokens[1:], "") - if r.RawRequest != nil { - // rawhttp - r.RawRequest.Headers[headerName] = headerValue - } else { - // retryablehttp - headerName = strings.TrimSpace(headerName) - headerValue = strings.TrimSpace(headerValue) - r.Request.Header[headerName] = []string{headerValue} - } - } -} diff --git a/v2/pkg/executer/output_http.go b/v2/pkg/executer/output_http.go deleted file mode 100644 index 7ce421136..000000000 --- a/v2/pkg/executer/output_http.go +++ /dev/null @@ -1,147 +0,0 @@ -package executer - -import ( - "fmt" - "net/http" - "net/http/httputil" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" -) - -// writeOutputHTTP writes http output to streams -func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}, reqURL string) { - var URL string - if req.RawRequest != nil { - URL = req.RawRequest.FullURL - } - if req.Request != nil { - URL = req.Request.URL.String() - } - - if e.jsonOutput { - output := make(jsonOutput) - - output["matched"] = URL - if !e.noMeta { - output["template"] = e.template.ID - output["type"] = "http" - output["host"] = reqURL - if len(meta) > 0 { - output["meta"] = meta - } - for k, v := range e.template.Info { - output[k] = v - } - if matcher != nil && len(matcher.Name) > 0 { - output["matcher_name"] = matcher.Name - } - if len(extractorResults) > 0 { - output["extracted_results"] = extractorResults - } - - // TODO: URL should be an argument - if e.jsonRequest { - dumpedRequest, err := requests.Dump(req, URL) - if err != nil { - gologger.Warningf("could not dump request: %s\n", err) - } else { - output["request"] = string(dumpedRequest) - } - - dumpedResponse, err := httputil.DumpResponse(resp, false) - if err != nil { - gologger.Warningf("could not dump response: %s\n", err) - } else { - output["response"] = string(dumpedResponse) + body - } - } - } - - data, err := jsoniter.Marshal(output) - if err != nil { - gologger.Warningf("Could not marshal json output: %s\n", err) - } - gologger.Silentf("%s", string(data)) - - if e.writer != nil { - if err := e.writer.Write(data); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } - return - } - - builder := &strings.Builder{} - colorizer := e.colorizer - - if !e.noMeta { - builder.WriteRune('[') - builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String()) - - if matcher != nil && len(matcher.Name) > 0 { - builder.WriteString(":") - builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String()) - } - - builder.WriteString("] [") - builder.WriteString(colorizer.Colorizer.BrightBlue("http").String()) - builder.WriteString("] ") - - if e.template.Info["severity"] != "" { - builder.WriteString("[") - builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"])) - builder.WriteString("] ") - } - } - builder.WriteString(URL) - - // If any extractors, write the results - if len(extractorResults) > 0 && !e.noMeta { - builder.WriteString(" [") - - for i, result := range extractorResults { - builder.WriteString(colorizer.Colorizer.BrightCyan(result).String()) - - if i != len(extractorResults)-1 { - builder.WriteRune(',') - } - } - - builder.WriteString("]") - } - - // write meta if any - if len(req.Meta) > 0 && !e.noMeta { - builder.WriteString(" [") - - var metas []string - for name, value := range req.Meta { - metas = append(metas, colorizer.Colorizer.BrightYellow(name).Bold().String()+"="+colorizer.Colorizer.BrightYellow(fmt.Sprint(value)).String()) - } - - builder.WriteString(strings.Join(metas, ",")) - builder.WriteString("]") - } - - builder.WriteRune('\n') - - // Write output to screen as well as any output file - message := builder.String() - gologger.Silentf("%s", message) - - if e.writer != nil { - if e.coloredOutput { - message = e.decolorizer.ReplaceAllString(message, "") - } - - if err := e.writer.WriteString(message); err != nil { - gologger.Errorf("Could not write output data: %s\n", err) - return - } - } -} diff --git a/v2/pkg/executer/utils.go b/v2/pkg/executer/utils.go deleted file mode 100644 index 50954e5a2..000000000 --- a/v2/pkg/executer/utils.go +++ /dev/null @@ -1,59 +0,0 @@ -package executer - -import ( - "net/http" - "net/url" - "strings" - "unsafe" -) - -type jsonOutput map[string]interface{} - -// unsafeToString converts byte slice to string with zero allocations -func unsafeToString(bs []byte) string { - return *(*string)(unsafe.Pointer(&bs)) -} - -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - - if i != len(values)-1 { - builder.WriteRune('\n') - builder.WriteString(header) - builder.WriteString(": ") - } - } - builder.WriteRune('\n') - } - return builder.String() -} - -// isURL tests a string to determine if it is a well-structured url or not. -func isURL(toTest string) bool { - _, err := url.ParseRequestURI(toTest) - if err != nil { - return false - } - u, err := url.Parse(toTest) - if err != nil || u.Scheme == "" || u.Host == "" { - return false - } - return true -} - -// extractDomain extracts the domain name of a URL -func extractDomain(theURL string) string { - u, err := url.Parse(theURL) - if err != nil { - return "" - } - return u.Hostname() -} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index b5f099411..971cb3a8c 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -32,11 +32,18 @@ var StringToType = map[string]Type{ } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, Type Type) (*Generator, error) { +func New(payloads map[string]interface{}, Type Type, templatePath string) (*Generator, error) { + generator := &Generator{} + if err := generator.validate(payloads, templatePath); err != nil { + return nil, err + } + compiled, err := loadPayloads(payloads) if err != nil { return nil, err } + generator.Type = Type + generator.payloads = compiled // Validate the payload types if Type == Sniper && len(compiled) > 1 { @@ -51,7 +58,6 @@ func New(payloads map[string]interface{}, Type Type) (*Generator, error) { totalLength = len(v) } } - generator := &Generator{Type: Type, payloads: compiled} return generator, nil } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 93843c856..1380dac90 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -9,7 +9,7 @@ import ( func TestSniperGenerator(t *testing.T) { usernames := []string{"admin", "password", "login", "test"} - generator, err := New(map[string]interface{}{"username": usernames}, Sniper) + generator, err := New(map[string]interface{}{"username": usernames}, Sniper, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -29,7 +29,7 @@ func TestPitchforkGenerator(t *testing.T) { usernames := []string{"admin", "token"} passwords := []string{"admin", "password"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchFork, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -50,7 +50,7 @@ func TestClusterbombGenerator(t *testing.T) { usernames := []string{"admin"} passwords := []string{"admin", "password", "token"} - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb) + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBomb, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go new file mode 100644 index 000000000..acc8a2da0 --- /dev/null +++ b/v2/pkg/protocols/common/generators/validate.go @@ -0,0 +1,61 @@ +package generators + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "github.com/spf13/cast" +) + +// validate validates the payloads if any. +func (g *Generator) validate(payloads map[string]interface{}, templatePath string) error { + for name, payload := range payloads { + switch pt := payload.(type) { + case string: + // check if it's a multiline string list + if len(strings.Split(pt, "\n")) != 1 { + return errors.New("invalid number of lines in payload") + } + + // check if it's a worldlist file and try to load it + if fileExists(pt) { + continue + } + + changed := false + pathTokens := strings.Split(templatePath, "/") + + for i := range pathTokens { + tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt) + if fileExists(tpath) { + payloads[name] = tpath + changed = true + break + } + } + if !changed { + return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) + } + case interface{}: + loadedPayloads := cast.ToStringSlice(pt) + if len(loadedPayloads) == 0 { + return fmt.Errorf("the payload %s does not contain enough elements", name) + } + default: + return fmt.Errorf("the payload %s has invalid type", name) + } + } + return nil +} + +// fileExists checks if a file exists and is not a directory +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/v2/pkg/protocols/dns/executer_test.go b/v2/pkg/protocols/dns/executer_test.go deleted file mode 100644 index 9143eabaf..000000000 --- a/v2/pkg/protocols/dns/executer_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package dns - -import ( - "fmt" - "testing" - - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" -) - -func TestRequest(t *testing.T) { - err := dnsclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - } - req := &Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, - }} - err = req.Compile(protocolOpts) - require.Nil(t, err, "could not compile request") - - output, err := req.ExecuteWithResults("docs.hackerone.com.", nil) - require.Nil(t, err, "could not execute request") - - for _, result := range output { - fmt.Printf("%+v\n", result) - } -} - -func TestExecuter(t *testing.T) { - err := dnsclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - } - executer := NewExecuter([]*Request{&Request{Name: "{{FQDN}}", Recursion: true, Class: "inet", Type: "CNAME", Retries: 5, Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "word", Words: []string{"github.io"}, Part: "body"}}, - }}}, protocolOpts) - err = executer.Compile() - require.Nil(t, err, "could not compile request") - - _, err = executer.Execute("docs.hackerone.com") - require.Nil(t, err, "could not execute request") -} diff --git a/v2/pkg/protocols/http/build_request_test.go b/v2/pkg/protocols/http/build_request_test.go index 846b8896c..b16f5c98b 100644 --- a/v2/pkg/protocols/http/build_request_test.go +++ b/v2/pkg/protocols/http/build_request_test.go @@ -31,7 +31,7 @@ func TestRequestGeneratorClusterSingle(t *testing.T) { attackType: generators.ClusterBomb, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType) + req.generator, err = generators.New(req.Payloads, req.attackType, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator() @@ -54,7 +54,7 @@ func TestRequestGeneratorClusterMultipleRaw(t *testing.T) { attackType: generators.ClusterBomb, Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } - req.generator, err = generators.New(req.Payloads, req.attackType) + req.generator, err = generators.New(req.Payloads, req.attackType, "") require.Nil(t, err, "could not create generator") generator := req.newGenerator() diff --git a/v2/pkg/protocols/http/executer_test.go b/v2/pkg/protocols/http/executer_test.go deleted file mode 100644 index a8bafdb26..000000000 --- a/v2/pkg/protocols/http/executer_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package http - -import ( - "testing" - - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" - "go.uber.org/ratelimit" -) - -func TestRequest(t *testing.T) { - err := httpclientpool.Init(&types.Options{}) - require.Nil(t, err, "could not initialize dns client pool") - - writer, err := output.NewStandardWriter(true, false, false, "", "") - require.Nil(t, err, "could not create standard output writer") - - progress, err := progress.NewProgress(false, false, 0) - require.Nil(t, err, "could not create standard progress writer") - - protocolOpts := &protocols.ExecuterOptions{ - TemplateID: "testing-dns", - TemplateInfo: map[string]string{"author": "test"}, - Output: writer, - Options: &types.Options{}, - Progress: progress, - RateLimiter: ratelimit.New(100), - } - executer := NewExecuter([]*Request{&Request{Path: []string{"{{BaseURL}}"}, Method: "GET", Operators: &operators.Operators{ - Matchers: []*matchers.Matcher{{Type: "dsl", DSL: []string{"!contains(tolower(all_headers), 'x-frame-options')"}, Part: "body"}}, - }}}, protocolOpts) - err = executer.Compile() - require.Nil(t, err, "could not compile request") - - _, err = executer.Execute("https://example.com") - require.Nil(t, err, "could not execute request") - - // for _, result := range output { - // fmt.Printf("%+v\n", result) - // } -} diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 949a210ba..00cb2131b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -91,8 +91,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } if len(r.Payloads) > 0 { - r.attackType = generators.StringToType[r.AttackType] - r.generator, err = generators.New(r.Payloads, r.attackType) + attackType := r.AttackType + if attackType == "" { + attackType = "sniper" + } + r.attackType = generators.StringToType[attackType] + + r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9be2ef1c1..c3c99973c 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -26,6 +26,8 @@ type Executer interface { type ExecuterOptions struct { // TemplateID is the ID of the template for the request TemplateID string + // TemplatePath is the path of the template for the request + TemplatePath string // TemplateInfo contains information block of the template request TemplateInfo map[string]string // Output is a writer interface for writing output events from executer. diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index f15d42b26..4ed579c2a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -3,11 +3,7 @@ package templates import ( "fmt" "os" - "path" - "strings" - "github.com/projectdiscovery/nuclei/v2/pkg/generators" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "gopkg.in/yaml.v2" ) @@ -35,99 +31,9 @@ func Parse(file string) (*Template, error) { // Compile the matchers and the extractors for http requests for _, request := range template.BulkRequestsHTTP { - // Get the condition between the matchers - condition, ok := matchers.ConditionTypes[request.MatchersCondition] - if !ok { - request.SetMatchersCondition(matchers.ORCondition) - } else { - request.SetMatchersCondition(condition) - } - - // Set the attack type - used only in raw requests - attack, ok := generators.AttackTypes[request.AttackType] - if !ok { - request.SetAttackType(generators.Sniper) - } else { - request.SetAttackType(attack) - } - - // Validate the payloads if any - for name, payload := range request.Payloads { - switch pt := payload.(type) { - case string: - // check if it's a multiline string list - if len(strings.Split(pt, "\n")) <= 1 { - // check if it's a worldlist file - if !generators.FileExists(pt) { - // attempt to load the file by taking the full path, tokezining it and searching the template in such paths - changed := false - pathTokens := strings.Split(template.path, "/") - - for i := range pathTokens { - tpath := path.Join(strings.Join(pathTokens[:i], "/"), pt) - if generators.FileExists(tpath) { - request.Payloads[name] = tpath - changed = true - - break - } - } - - if !changed { - return nil, fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) - } - } - } - case []string, []interface{}: - if len(payload.([]interface{})) == 0 { - return nil, fmt.Errorf("the payload %s does not contain enough elements", name) - } - default: - return nil, fmt.Errorf("the payload %s has invalid type", name) - } - } - - for _, matcher := range request.Matchers { - matchErr := matcher.CompileMatchers() - if matchErr != nil { - return nil, matchErr - } - } - - for _, extractor := range request.Extractors { - extractErr := extractor.CompileExtractors() - if extractErr != nil { - return nil, extractErr - } - } request.InitGenerator() } - // Compile the matchers and the extractors for dns requests - for _, request := range template.RequestsDNS { - // Get the condition between the matchers - condition, ok := matchers.ConditionTypes[request.MatchersCondition] - if !ok { - request.SetMatchersCondition(matchers.ORCondition) - } else { - request.SetMatchersCondition(condition) - } - - for _, matcher := range request.Matchers { - err = matcher.CompileMatchers() - if err != nil { - return nil, err - } - } - - for _, extractor := range request.Extractors { - err := extractor.CompileExtractors() - if err != nil { - return nil, err - } - } - } - return template, nil } From 62603b7d5f8f4a8f074f042aa825ee911bca8c68 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 15:38:14 +0530 Subject: [PATCH 35/92] Making nuclei overall compatible with new changes + bug fixes --- v2/internal/colorizer/colorizer.go | 25 +++++ v2/internal/runner/banner.go | 8 +- v2/internal/runner/config.go | 5 +- v2/internal/runner/options.go | 49 +++++----- v2/internal/runner/paths.go | 8 +- v2/internal/runner/processor.go | 25 +---- v2/internal/runner/runner.go | 146 ++++++++++------------------- v2/internal/runner/templates.go | 129 ++++++++++--------------- v2/internal/runner/update.go | 38 ++------ v2/pkg/output/format_screen.go | 2 +- v2/pkg/output/output.go | 47 ++++------ v2/pkg/protocols/dns/dns.go | 2 +- v2/pkg/protocols/dns/executer.go | 4 +- v2/pkg/protocols/http/executer.go | 4 +- v2/pkg/protocols/protocols.go | 4 +- v2/pkg/templates/compile.go | 27 ++++-- v2/pkg/templates/templates.go | 38 ++++---- v2/pkg/workflows/compile.go | 2 - v2/pkg/workflows/execute.go | 2 +- v2/pkg/workflows/execute_test.go | 2 +- v2/pkg/workflows/workflows.go | 11 --- 21 files changed, 225 insertions(+), 353 deletions(-) create mode 100644 v2/internal/colorizer/colorizer.go diff --git a/v2/internal/colorizer/colorizer.go b/v2/internal/colorizer/colorizer.go new file mode 100644 index 000000000..0b001b79b --- /dev/null +++ b/v2/internal/colorizer/colorizer.go @@ -0,0 +1,25 @@ +package colorizer + +import "github.com/logrusorgru/aurora" + +// Colorizer returns a colorized severity printer +type Colorizer struct { + Data map[string]string +} + +const ( + fgOrange uint8 = 208 + undefined string = "undefined" +) + +// New returns a new severity based colorizer +func New(colorizer aurora.Aurora) *Colorizer { + severityMap := map[string]string{ + "info": colorizer.Blue("info").String(), + "low": colorizer.Green("low").String(), + "medium": colorizer.Yellow("medium").String(), + "high": colorizer.Index(fgOrange, "high").String(), + "critical": colorizer.Red("critical").String(), + } + return &Colorizer{Data: severityMap} +} diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 88a1de78d..74ab34e51 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -15,9 +15,9 @@ const Version = `2.2.1-dev` // showBanner is used to show the banner to the user func showBanner() { - gologger.Printf("%s\n", banner) - gologger.Printf("\t\tprojectdiscovery.io\n\n") + gologger.Print().Msgf("%s\n", banner) + gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Labelf("Use with caution. You are responsible for your actions\n") - gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Warning().Msgf("Use with caution. You are responsible for your actions\n") + gologger.Warning().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index 73efccb80..db739dd50 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -105,7 +105,6 @@ func (r *Runner) checkIfInNucleiIgnore(item string) bool { if strings.Contains(item, paths) { return true } - continue } // Check for file based extension in ignores @@ -113,10 +112,10 @@ func (r *Runner) checkIfInNucleiIgnore(item string) bool { return true } } - return false } +// getIgnoreFilePath returns the ignore file path for the runner func (r *Runner) getIgnoreFilePath() string { defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile) @@ -124,13 +123,11 @@ func (r *Runner) getIgnoreFilePath() string { if err != nil { return defIgnoreFilePath } - cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile) cwdIfpInfo, err := os.Stat(cwdIgnoreFilePath) if os.IsNotExist(err) || cwdIfpInfo.IsDir() { return defIgnoreFilePath } - return cwdIgnoreFilePath } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index a7b66dee2..5186db62e 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -7,13 +7,15 @@ import ( "os" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/formatter" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // ParseOptions parses the command line flags provided by a user -func ParseOptions() *Options { - options := &Options{} +func ParseOptions() *types.Options { + options := &types.Options{} - flag.BoolVar(&options.Vhost, "vhost", false, "Input supplied is a comma-separated vhost list") flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") @@ -57,34 +59,34 @@ func ParseOptions() *Options { options.Stdin = hasStdin() // Read the inputs and configure the logging - options.configureOutput() + configureOutput(options) // Show the user the banner showBanner() if options.Version { - gologger.Infof("Current Version: %s\n", Version) + gologger.Info().Msgf("Current Version: %s\n", Version) os.Exit(0) } if options.TemplatesVersion { config, err := readConfiguration() if err != nil { - gologger.Fatalf("Could not read template configuration: %s\n", err) + gologger.Fatal().Msgf("Could not read template configuration: %s\n", err) } - gologger.Infof("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory) + gologger.Info().Msgf("Current nuclei-templates version: %s (%s)\n", config.CurrentVersion, config.TemplatesDirectory) os.Exit(0) } // Validate the options passed by the user and if any // invalid options have been used, exit. - err := options.validateOptions() + err := validateOptions(options) if err != nil { - gologger.Fatalf("Program exiting: %s\n", err) + gologger.Fatal().Msgf("Program exiting: %s\n", err) } - return options } +// hasStdin returns true if we have stdin input func hasStdin() bool { stat, err := os.Stdin.Stat() if err != nil { @@ -98,7 +100,7 @@ func hasStdin() bool { } // validateOptions validates the configuration options passed -func (options *Options) validateOptions() error { +func validateOptions(options *types.Options) error { // Both verbose and silent flags were used if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") @@ -116,22 +118,15 @@ func (options *Options) validateOptions() error { } // Validate proxy options if provided - err := validateProxyURL( - options.ProxyURL, - "invalid http proxy format (It should be http://username:password@host:port)", - ) + err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)") if err != nil { return err } - err = validateProxyURL( - options.ProxySocksURL, - "invalid socks proxy format (It should be socks5://username:password@host:port)", - ) + err = validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)") if err != nil { return err } - return nil } @@ -145,22 +140,22 @@ func validateProxyURL(proxyURL, message string) error { func isValidURL(urlString string) bool { _, err := url.Parse(urlString) - return err == nil } // configureOutput configures the output on the screen -func (options *Options) configureOutput() { +func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output if options.Verbose { - gologger.MaxLevel = gologger.Verbose + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } + if options.Debug { + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) } - if options.NoColor { - gologger.UseColors = false + gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) } - if options.Silent { - gologger.MaxLevel = gologger.Silent + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } } diff --git a/v2/internal/runner/paths.go b/v2/internal/runner/paths.go index 4740142aa..be4b42a33 100644 --- a/v2/internal/runner/paths.go +++ b/v2/internal/runner/paths.go @@ -14,7 +14,6 @@ func isRelative(filePath string) bool { if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") { return false } - return true } @@ -30,19 +29,16 @@ func (r *Runner) resolvePath(templateName string) (string, error) { templatePath := path.Join(curDirectory, templateName) if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debugf("Found template in current directory: %s\n", templatePath) - + gologger.Debug().Msgf("Found template in current directory: %s\n", templatePath) return templatePath, nil } if r.templatesConfig != nil { templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName) if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debugf("Found template in nuclei-templates directory: %s\n", templatePath) - + gologger.Debug().Msgf("Found template in nuclei-templates directory: %s\n", templatePath) return templatePath, nil } } - return "", fmt.Errorf("no such path found: %s", templateName) } diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 68d4e5683..82fbb46f7 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -1,36 +1,20 @@ package runner import ( - "context" "fmt" - "net/http/cookiejar" "os" "path" - "path/filepath" - "strings" - "time" - tengo "github.com/d5/tengo/v2" - "github.com/d5/tengo/v2/stdlib" - "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" - "github.com/projectdiscovery/nuclei/v2/pkg/executer" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/workflows" - "github.com/remeh/sizedwaitgroup" ) +/* // workflowTemplates contains the initialized workflow templates per template group type workflowTemplates struct { Name string Templates []*workflows.Template } -var sandboxedModules = []string{"math", "text", "rand", "fmt", "json", "base64", "hex", "enum"} - // processTemplateWithList processes a template and runs the enumeration on all the targets func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { var httpExecuter *executer.HTTPExecuter @@ -332,16 +316,15 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl } wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst}) } - return &wflTemplatesList, nil -} +}*/ +// resolvePathWithBaseFolder resolves a path with the base folder func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) { templatePath := path.Join(baseFolder, templateName) if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debugf("Found template in current directory: %s\n", templatePath) + gologger.Debug().Msgf("Found template in current directory: %s\n", templatePath) return templatePath, nil } - return "", fmt.Errorf("no such path found: %s", templateName) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 79ded4bf0..d7e8ddd7b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -3,22 +3,19 @@ package runner import ( "bufio" "os" - "regexp" "strings" "github.com/logrusorgru/aurora" - "github.com/pkg/errors" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" - "github.com/projectdiscovery/nuclei/v2/internal/bufwriter" + "github.com/projectdiscovery/nuclei/v2/internal/collaborator" + "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/internal/tracelog" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" - "github.com/projectdiscovery/nuclei/v2/pkg/collaborator" - "github.com/projectdiscovery/nuclei/v2/pkg/colorizer" + "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/remeh/sizedwaitgroup" "go.uber.org/ratelimit" @@ -26,60 +23,31 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - inputCount int64 - - traceLog tracelog.Log - - // output is the output file to write if any - output *bufwriter.Writer - + hostMap *hybrid.HybridMap + output output.Writer + inputCount int64 templatesConfig *nucleiConfig - // options contains configuration options for runner - options *Options - - pf *projectfile.ProjectFile - - // progress tracking - progress *progress.Progress - - // output coloring - colorizer colorizer.NucleiColorizer - decolorizer *regexp.Regexp - - // rate limiter - ratelimiter ratelimit.Limiter - - // input deduplication - hm *hybrid.HybridMap - dialer *fastdialer.Dialer + options *types.Options + projectFile *projectfile.ProjectFile + progress *progress.Progress + colorizer aurora.Aurora + severityColors *colorizer.Colorizer + ratelimiter ratelimit.Limiter } // New creates a new client for running enumeration process. -func New(options *Options) (*Runner, error) { +func New(options *types.Options) (*Runner, error) { runner := &Runner{ - traceLog: &tracelog.NoopLogger{}, - options: options, + options: options, } - if options.TraceLogFile != "" { - fileLog, err := tracelog.NewFileLogger(options.TraceLogFile) - if err != nil { - return nil, errors.Wrap(err, "could not create file trace logger") - } - runner.traceLog = fileLog - } - if err := runner.updateTemplates(); err != nil { - gologger.Labelf("Could not update templates: %s\n", err) + gologger.Warning().Msgf("Could not update templates: %s\n", err) } // output coloring useColor := !options.NoColor - runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor)) - - if useColor { - // compile a decolorization regex to cleanup file output messages - runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) - } + runner.colorizer = aurora.NewAurora(useColor) + runner.severityColors = colorizer.New(runner.colorizer) if options.TemplateList { runner.listAvailableTemplates() @@ -95,9 +63,9 @@ func New(options *Options) (*Runner, error) { } if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil { - gologger.Fatalf("Could not create temporary input file: %s\n", err) + gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err) } else { - runner.hm = hm + runner.hostMap = hm } runner.inputCount = 0 @@ -107,7 +75,7 @@ func New(options *Options) (*Runner, error) { if options.Target != "" { runner.inputCount++ // nolint:errcheck // ignoring error - runner.hm.Set(options.Target, nil) + runner.hostMap.Set(options.Target, nil) } // Handle stdin @@ -115,20 +83,16 @@ func New(options *Options) (*Runner, error) { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { url := strings.TrimSpace(scanner.Text()) - // skip empty lines if url == "" { continue } - - // skip dupes - if _, ok := runner.hm.Get(url); ok { + if _, ok := runner.hostMap.Get(url); ok { dupeCount++ continue } - runner.inputCount++ // nolint:errcheck // ignoring error - runner.hm.Set(url, nil) + runner.hostMap.Set(url, nil) } } @@ -136,38 +100,34 @@ func New(options *Options) (*Runner, error) { if options.Targets != "" { input, err := os.Open(options.Targets) if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) + gologger.Fatal().Msgf("Could not open targets file '%s': %s\n", options.Targets, err) } scanner := bufio.NewScanner(input) for scanner.Scan() { url := strings.TrimSpace(scanner.Text()) - // skip empty lines if url == "" { continue } - - // skip dupes - if _, ok := runner.hm.Get(url); ok { + if _, ok := runner.hostMap.Get(url); ok { dupeCount++ continue } - runner.inputCount++ // nolint:errcheck // ignoring error - runner.hm.Set(url, nil) + runner.hostMap.Set(url, nil) } input.Close() } if dupeCount > 0 { - gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount) + gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", dupeCount) } // Create the output file if asked if options.Output != "" { - output, errBufWriter := bufwriter.New(options.Output) - if errBufWriter != nil { - gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter) + output, errWriter := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile) + if errWriter != nil { + gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, errWriter) } runner.output = output } @@ -182,7 +142,7 @@ func New(options *Options) (*Runner, error) { // create project file if requested or load existing one if options.Project { var projectFileErr error - runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""}) + runner.projectFile, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""}) if projectFileErr != nil { return nil, projectFileErr } @@ -193,19 +153,11 @@ func New(options *Options) (*Runner, error) { collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid) } - // Create Dialer - var err error - runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions) - if err != nil { - return nil, err - } - if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { runner.ratelimiter = ratelimit.NewUnlimited() } - return runner, nil } @@ -214,9 +166,9 @@ func (r *Runner) Close() { if r.output != nil { r.output.Close() } - r.hm.Close() - if r.pf != nil { - r.pf.Close() + r.hostMap.Close() + if r.projectFile != nil { + r.projectFile.Close() } } @@ -241,7 +193,7 @@ func (r *Runner) RunEnumeration() { if _, found := excludedMap[incl]; !found { allTemplates = append(allTemplates, incl) } else { - gologger.Warningf("Excluding '%s'", incl) + gologger.Warning().Msgf("Excluding '%s'", incl) } } } @@ -253,25 +205,24 @@ func (r *Runner) RunEnumeration() { // 0 matches means no templates were found in directory if templateCount == 0 { - gologger.Fatalf("Error, no templates were found.\n") + gologger.Fatal().Msgf("Error, no templates were found.\n") } - gologger.Infof("Using %s rules (%s templates, %s workflows)", - r.colorizer.Colorizer.Bold(templateCount).String(), - r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(), - r.colorizer.Colorizer.Bold(workflowCount).String()) + gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)", + r.colorizer.Bold(templateCount).String(), + r.colorizer.Bold(templateCount-workflowCount).String(), + r.colorizer.Bold(workflowCount).String()) // precompute total request count var totalRequests int64 = 0 for _, t := range availableTemplates { - switch av := t.(type) { - case *templates.Template: - totalRequests += (av.GetHTTPRequestCount() + av.GetDNSRequestCount()) * r.inputCount - case *workflows.Workflow: - // workflows will dynamically adjust the totals while running, as - // it can't be know in advance which requests will be called - } // nolint:wsl // comment + // workflows will dynamically adjust the totals while running, as + // it can't be know in advance which requests will be called + if t.Workflow != nil { + continue + } + totalRequests += int64(t.Requests()) * r.inputCount } results := atomicboolean.New() @@ -280,7 +231,7 @@ func (r *Runner) RunEnumeration() { collaborator.DefaultCollaborator.Poll() if r.inputCount == 0 { - gologger.Errorf("Could not find any valid input URLs.") + gologger.Error().Msgf("Could not find any valid input URLs.") } else if totalRequests > 0 || hasWorkflows { // tracks global progress and captures stdout/stderr until p.Wait finishes p := r.progress @@ -313,7 +264,6 @@ func (r *Runner) RunEnumeration() { r.output.Close() os.Remove(r.options.Output) } - - gologger.Infof("No results found. Happy hacking!") + gologger.Info().Msgf("No results found. Happy hacking!") } } diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index f133b390b..b8a11d8f1 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -1,7 +1,6 @@ package runner import ( - "errors" "fmt" "os" "path/filepath" @@ -9,8 +8,8 @@ import ( "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) // getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths. @@ -22,7 +21,6 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { // parses user input, handle file/directory cases and produce a list of unique templates for _, t := range definitions { var absPath string - var err error if strings.Contains(t, "*") { @@ -34,9 +32,8 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { // resolve and convert relative to absolute path absPath, err = r.resolvePathIfRelative(t) } - if err != nil { - gologger.Errorf("Could not find template file '%s': %s\n", t, err) + gologger.Error().Msgf("Could not find template file '%s': %s\n", t, err) continue } @@ -44,25 +41,22 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { if strings.Contains(absPath, "*") { var matches []string matches, err = filepath.Glob(absPath) - if err != nil { - gologger.Labelf("Wildcard found, but unable to glob '%s': %s\n", absPath, err) - + gologger.Error().Msgf("Wildcard found, but unable to glob '%s': %s\n", absPath, err) continue } // couldn't find templates in directory if len(matches) == 0 { - gologger.Labelf("Error, no templates were found with '%s'.\n", absPath) + gologger.Error().Msgf("Error, no templates were found with '%s'.\n", absPath) continue } else { - gologger.Labelf("Identified %d templates\n", len(matches)) + gologger.Verbose().Msgf("Identified %d templates\n", len(matches)) } for _, match := range matches { if !r.checkIfInNucleiIgnore(match) { processed[match] = true - allTemplates = append(allTemplates, match) } } @@ -70,7 +64,7 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { // determine file/directory isFile, err := isFilePath(absPath) if err != nil { - gologger.Errorf("Could not stat '%s': %s\n", absPath, err) + gologger.Error().Msgf("Could not stat '%s': %s\n", absPath, err) continue } // test for uniqueness @@ -88,8 +82,7 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { matches := []string{} // Recursively walk down the Templates directory and run all the template file checks - err := directoryWalker( - absPath, + err := directoryWalker(absPath, func(path string, d *godirwalk.Dirent) error { if !d.IsDir() && strings.HasSuffix(path, ".yaml") { if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) { @@ -100,111 +93,89 @@ func (r *Runner) getTemplatesFor(definitions []string) []string { return nil }, ) - // directory couldn't be walked if err != nil { - gologger.Labelf("Could not find templates in directory '%s': %s\n", absPath, err) + gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", absPath, err) continue } // couldn't find templates in directory if len(matches) == 0 { - gologger.Labelf("Error, no templates were found in '%s'.\n", absPath) + gologger.Error().Msgf("Error, no templates were found in '%s'.\n", absPath) continue } - allTemplates = append(allTemplates, matches...) } } } - return allTemplates } // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. -func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []interface{}, workflowCount int) { +func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []*templates.Template, workflowCount int) { workflowCount = 0 severities = strings.ToLower(severities) allSeverities := strings.Split(severities, ",") filterBySeverity := len(severities) > 0 - gologger.Infof("Loading templates...") + gologger.Info().Msgf("Loading templates...") for _, match := range templatePaths { t, err := r.parseTemplateFile(match) - switch tp := t.(type) { - case *templates.Template: - // only include if severity matches or no severity filtering - sev := strings.ToLower(tp.Info["severity"]) - if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) { - parsedTemplates = append(parsedTemplates, tp) - gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"])) - } else { - gologger.Warningf("Excluding template %s due to severity filter (%s not in [%s])", tp.ID, sev, severities) + if err != nil { + gologger.Error().Msgf("Could not parse file '%s': %s\n", match, err) + continue + } + sev := strings.ToLower(t.Info["severity"]) + if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) { + parsedTemplates = append(parsedTemplates, t) + // Process the template like a workflow + if t.Workflow != nil { + workflowCount++ } - case *workflows.Workflow: - parsedTemplates = append(parsedTemplates, tp) - gologger.Infof("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"])) - workflowCount++ - default: - gologger.Errorf("Could not parse file '%s': %s\n", match, err) + gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) + } else { + gologger.Error().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities) } } - return parsedTemplates, workflowCount } -func (r *Runner) parseTemplateFile(file string) (interface{}, error) { - // check if it's a template - template, errTemplate := templates.Parse(file) - if errTemplate == nil { - return template, nil +// parseTemplateFile returns the parsed template file +func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { + executerOpts := &protocols.ExecuterOptions{ + Output: r.output, + Options: r.options, + Progress: r.progress, + RateLimiter: r.ratelimiter, + ProjectFile: r.projectFile, } - - // check if it's a workflow - workflow, errWorkflow := workflows.Parse(file) - if errWorkflow == nil { - return workflow, nil + template, err := templates.Parse(file, executerOpts) + if err != nil { + return nil, err } - - if errTemplate != nil { - return nil, errTemplate - } - - if errWorkflow != nil { - return nil, errWorkflow - } - - return nil, errors.New("unknown error occurred") + return template, nil } func (r *Runner) templateLogMsg(id, name, author, severity string) string { // Display the message for the template message := fmt.Sprintf("[%s] %s (%s)", - r.colorizer.Colorizer.BrightBlue(id).String(), - r.colorizer.Colorizer.Bold(name).String(), - r.colorizer.Colorizer.BrightYellow("@"+author).String()) - + r.colorizer.BrightBlue(id).String(), + r.colorizer.Bold(name).String(), + r.colorizer.BrightYellow("@"+author).String()) if severity != "" { - message += " [" + r.colorizer.GetColorizedSeverity(severity) + "]" + message += " [" + r.severityColors.Data[severity] + "]" } - return message } func (r *Runner) logAvailableTemplate(tplPath string) { t, err := r.parseTemplateFile(tplPath) - if t != nil { - switch tp := t.(type) { - case *templates.Template: - gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"])) - case *workflows.Workflow: - gologger.Silentf("%s\n", r.templateLogMsg(tp.ID, tp.Info["name"], tp.Info["author"], tp.Info["severity"])) - default: - gologger.Errorf("Could not parse file '%s': %s\n", tplPath, err) - } + if err != nil { + gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err) } + gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } // ListAvailableTemplates prints available templates to stdout @@ -214,11 +185,11 @@ func (r *Runner) listAvailableTemplates() { } if _, err := os.Stat(r.templatesConfig.TemplatesDirectory); os.IsNotExist(err) { - gologger.Errorf("%s does not exists", r.templatesConfig.TemplatesDirectory) + gologger.Error().Msgf("%s does not exists", r.templatesConfig.TemplatesDirectory) return } - gologger.Silentf( + gologger.Print().Msgf( "\nListing available v.%s nuclei templates for %s", r.templatesConfig.CurrentVersion, r.templatesConfig.TemplatesDirectory, @@ -227,18 +198,16 @@ func (r *Runner) listAvailableTemplates() { r.templatesConfig.TemplatesDirectory, func(path string, d *godirwalk.Dirent) error { if d.IsDir() && path != r.templatesConfig.TemplatesDirectory { - gologger.Silentf("\n%s:\n\n", r.colorizer.Colorizer.Bold(r.colorizer.Colorizer.BgBrightBlue(d.Name())).String()) + gologger.Print().Msgf("\n%s:\n\n", r.colorizer.Bold(r.colorizer.BgBrightBlue(d.Name())).String()) } else if strings.HasSuffix(path, ".yaml") { r.logAvailableTemplate(path) } - return nil }, ) - // directory couldn't be walked if err != nil { - gologger.Labelf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err) + gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", r.templatesConfig.TemplatesDirectory, err) } } @@ -249,7 +218,6 @@ func (r *Runner) resolvePathIfRelative(filePath string) (string, error) { if err != nil { return "", err } - return newPath, nil } @@ -294,9 +262,8 @@ func isFilePath(filePath string) (bool, error) { func isNewPath(filePath string, pathMap map[string]bool) bool { if _, already := pathMap[filePath]; already { - gologger.Warningf("Skipping already specified path '%s'", filePath) + gologger.Warning().Msgf("Skipping already specified path '%s'", filePath) return false } - return true } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 0ec6adfbc..1c8d9f621 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -47,10 +47,9 @@ func (r *Runner) updateTemplates() error { } ctx := context.Background() - if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { if !r.options.UpdateTemplates { - gologger.Labelf("nuclei-templates are not installed, use update-templates flag.\n") + gologger.Warning().Msgf("nuclei-templates are not installed, use update-templates flag.\n") return nil } @@ -58,7 +57,6 @@ func (r *Runner) updateTemplates() error { if r.options.TemplatesDirectory != "" { home = r.options.TemplatesDirectory } - r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")} // Download the repository and also write the revision to a HEAD file. @@ -66,23 +64,19 @@ func (r *Runner) updateTemplates() error { if getErr != nil { return getErr } - - gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL()) if err != nil { return err } - r.templatesConfig.CurrentVersion = version.String() err = r.writeConfiguration(r.templatesConfig) if err != nil { return err } - - gologger.Infof("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String()) - + gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). Enjoy!\n", version.String()) return nil } @@ -95,17 +89,14 @@ func (r *Runner) updateTemplates() error { // Get the configuration currently on disk. verText := r.templatesConfig.CurrentVersion indices := reVersion.FindStringIndex(verText) - if indices == nil { return fmt.Errorf("invalid release found with tag %s", err) } - if indices[0] > 0 { verText = verText[indices[0]:] } oldVersion, err := semver.Make(verText) - if err != nil { return err } @@ -116,13 +107,13 @@ func (r *Runner) updateTemplates() error { } if version.EQ(oldVersion) { - gologger.Infof("Your nuclei-templates are up to date: v%s\n", oldVersion.String()) + gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String()) return r.writeConfiguration(r.templatesConfig) } if version.GT(oldVersion) { if !r.options.UpdateTemplates { - gologger.Labelf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) + gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) return r.writeConfiguration(r.templatesConfig) } @@ -130,11 +121,9 @@ func (r *Runner) updateTemplates() error { home = r.options.TemplatesDirectory r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates") } - r.templatesConfig.CurrentVersion = version.String() - gologger.Verbosef("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) - + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL()) if err != nil { return err @@ -144,10 +133,8 @@ func (r *Runner) updateTemplates() error { if err != nil { return err } - - gologger.Infof("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String()) + gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). Enjoy!\n", version.String()) } - return nil } @@ -162,17 +149,13 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor // Find the most recent version based on semantic versioning. var latestRelease semver.Version - var latestPublish *github.RepositoryRelease - for _, release := range rels { verText := release.GetTagName() indices := reVersion.FindStringIndex(verText) - if indices == nil { return semver.Version{}, nil, fmt.Errorf("invalid release found with tag %s", err) } - if indices[0] > 0 { verText = verText[indices[0]:] } @@ -187,11 +170,9 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor latestPublish = release } } - if latestPublish == nil { return semver.Version{}, nil, errors.New("no version found for the templates") } - return latestRelease, latestPublish, nil } @@ -207,7 +188,6 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string return fmt.Errorf("failed to download a release file from %s: %s", downloadURL, err) } defer res.Body.Close() - if res.StatusCode != http.StatusOK { return fmt.Errorf("failed to download a release file from %s: Not successful status %d", downloadURL, res.StatusCode) } @@ -219,7 +199,6 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string reader := bytes.NewReader(buf) z, err := zip.NewReader(reader, reader.Size()) - if err != nil { return fmt.Errorf("failed to uncompress zip file: %s", err) } @@ -241,7 +220,6 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath) err = os.MkdirAll(templateDirectory, os.ModePerm) - if err != nil { return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } @@ -263,9 +241,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string f.Close() return fmt.Errorf("could not write template file: %s", err) } - f.Close() } - return nil } diff --git a/v2/pkg/output/format_screen.go b/v2/pkg/output/format_screen.go index dc2860554..b46f24061 100644 --- a/v2/pkg/output/format_screen.go +++ b/v2/pkg/output/format_screen.go @@ -24,7 +24,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) ([]byte, error) { builder.WriteString("] ") builder.WriteString("[") - builder.WriteString(w.severityMap[output.Info["severity"]]) + builder.WriteString(w.severityColors.Data[output.Info["severity"]]) builder.WriteString("] ") } builder.WriteString(output.Matched) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 143bccab1..97b546967 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -8,6 +8,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" ) @@ -25,21 +26,16 @@ type Writer interface { // StandardWriter is a writer writing output to file and screen for results. type StandardWriter struct { - json bool - noMetadata bool - aurora aurora.Aurora - outputFile *fileWriter - outputMutex *sync.Mutex - traceFile *fileWriter - traceMutex *sync.Mutex - severityMap map[string]string + json bool + noMetadata bool + aurora aurora.Aurora + outputFile *fileWriter + outputMutex *sync.Mutex + traceFile *fileWriter + traceMutex *sync.Mutex + severityColors *colorizer.Colorizer } -const ( - fgOrange uint8 = 208 - undefined string = "undefined" -) - var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) // InternalEvent is an internal output generation structure for nuclei. @@ -78,7 +74,7 @@ type ResultEvent struct { // NewStandardWriter creates a new output writer based on user configurations func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (*StandardWriter, error) { - colorizer := aurora.NewAurora(colors) + auroraColorizer := aurora.NewAurora(colors) var outputFile *fileWriter if file != "" { @@ -96,22 +92,15 @@ func NewStandardWriter(colors, noMetadata, json bool, file, traceFile string) (* } traceOutput = output } - severityMap := map[string]string{ - "info": colorizer.Blue("info").String(), - "low": colorizer.Green("low").String(), - "medium": colorizer.Yellow("medium").String(), - "high": colorizer.Index(fgOrange, "high").String(), - "critical": colorizer.Red("critical").String(), - } writer := &StandardWriter{ - json: json, - noMetadata: noMetadata, - severityMap: severityMap, - aurora: colorizer, - outputFile: outputFile, - outputMutex: &sync.Mutex{}, - traceFile: traceOutput, - traceMutex: &sync.Mutex{}, + json: json, + noMetadata: noMetadata, + aurora: auroraColorizer, + outputFile: outputFile, + outputMutex: &sync.Mutex{}, + traceFile: traceOutput, + traceMutex: &sync.Mutex{}, + severityColors: colorizer.New(auroraColorizer), } return writer, nil } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 304ead0eb..4512fe795 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -60,7 +60,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int64 { +func (r *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go index ec919cfc5..7863efa0a 100644 --- a/v2/pkg/protocols/dns/executer.go +++ b/v2/pkg/protocols/dns/executer.go @@ -30,8 +30,8 @@ func (e *Executer) Compile() error { } // Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int64 { - var count int64 +func (e *Executer) Requests() int { + var count int for _, request := range e.requests { count += request.Requests() } diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go index c4c8724c7..fdcdb7586 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/http/executer.go @@ -30,8 +30,8 @@ func (e *Executer) Compile() error { } // Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int64 { - var count int64 +func (e *Executer) Requests() int { + var count int for _, request := range e.requests { count += int64(request.Requests()) } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index c3c99973c..67ea9e21d 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -15,7 +15,7 @@ type Executer interface { // Compile compiles the execution generators preparing any requests possible. Compile() error // Requests returns the total number of requests the rule will perform - Requests() int64 + Requests() int // Execute executes the protocol group and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. @@ -47,7 +47,7 @@ type Request interface { // Compile compiles the request generators preparing any requests possible. Compile(options *ExecuterOptions) error // Requests returns the total number of requests the rule will perform - Requests() int64 + Requests() int // Match performs matching operation for a matcher on model and returns true or false. Match(data map[string]interface{}, matcher *matchers.Matcher) bool // Extract performs extracting operation for a extractor on model and returns true or false. diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 4ed579c2a..4fc5d0bf6 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,11 +4,13 @@ import ( "fmt" "os" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "gopkg.in/yaml.v2" ) // Parse parses a yaml request template file -func Parse(file string) (*Template, error) { +func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { template := &Template{} f, err := os.Open(file) @@ -22,18 +24,29 @@ func Parse(file string) (*Template, error) { } defer f.Close() + // Setting up variables regarding template metadata template.path = file + options.TemplateID = template.ID + options.TemplateInfo = template.Info + options.TemplatePath = file // If no requests, and it is also not a workflow, return error. - if len(template.BulkRequestsHTTP)+len(template.RequestsDNS) <= 0 { + if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) <= 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } - // Compile the matchers and the extractors for http requests - for _, request := range template.BulkRequestsHTTP { - - request.InitGenerator() + // Compile the requests found + for _, request := range template.RequestsDNS { + if err := request.Compile(options); err != nil { + return nil, errors.Wrap(err, "could not compile dns request") + } + template.totalRequests += request.Requests() + } + for _, request := range template.RequestsHTTP { + if err := request.Compile(options); err != nil { + return nil, errors.Wrap(err, "could not compile dns request") + } + template.totalRequests += request.Requests() } - return template, nil } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 4b2ebfdce..41b7ff018 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -1,7 +1,9 @@ package templates import ( - "github.com/projectdiscovery/nuclei/v2/pkg/requests" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) // Template is a request template parsed from a yaml file @@ -10,32 +12,24 @@ type Template struct { ID string `yaml:"id"` // Info contains information about the template Info map[string]string `yaml:"info"` - // BulkRequestsHTTP contains the http request to make in the template - BulkRequestsHTTP []*requests.BulkHTTPRequest `yaml:"requests,omitempty"` + // RequestsHTTP contains the http request to make in the template + RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template - RequestsDNS []*requests.DNSRequest `yaml:"dns,omitempty"` - path string + RequestsDNS []*dns.Request `yaml:"dns,omitempty"` + + // Workflows is a yaml based workflow declaration code. + *workflows.Workflow + + path string + totalRequests int } -// GetPath of the workflow +// GetPath returns the path of the template. func (t *Template) GetPath() string { return t.path } -func (t *Template) GetHTTPRequestCount() int64 { - var count int64 = 0 - for _, request := range t.BulkRequestsHTTP { - count += request.GetRequestCount() - } - - return count -} - -func (t *Template) GetDNSRequestCount() int64 { - var count int64 = 0 - for _, request := range t.RequestsDNS { - count += request.GetRequestCount() - } - - return count +// Requests returns the number of requests for the template +func (t *Template) Requests() int { + return t.totalRequests } diff --git a/v2/pkg/workflows/compile.go b/v2/pkg/workflows/compile.go index ae40ba53e..3e973d8b7 100644 --- a/v2/pkg/workflows/compile.go +++ b/v2/pkg/workflows/compile.go @@ -25,7 +25,5 @@ func Parse(file string) (*Workflow, error) { if len(workflow.Workflows) == 0 { return nil, errors.New("no workflow defined") } - // Compile workflow here. - workflow.path = file return workflow, nil } diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index b51f347ac..5dc43e0ff 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -3,7 +3,7 @@ package workflows import "go.uber.org/atomic" // RunWorkflow runs a workflow on an input and returns true or false -func (w *Workflow) RunWorkflow(input string) (bool, error) { +func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) { results := &atomic.Bool{} for _, template := range w.Workflows { diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 2822863c0..47120be2e 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -149,7 +149,7 @@ func (m *mockExecuter) Compile() error { } // Requests returns the total number of requests the rule will perform -func (m *mockExecuter) Requests() int64 { +func (m *mockExecuter) Requests() int { return 1 } diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index 8edc313c0..f69cdcaf1 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -4,14 +4,8 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" // Workflow is a workflow to execute with chained requests, etc. type Workflow struct { - // ID is the unique id for the template - ID string `yaml:"id"` - // Info contains information about the template - Info map[string]string `yaml:"info"` // Workflows is a yaml based workflow declaration code. Workflows []*WorkflowTemplate `yaml:"workflows"` - - path string } // WorkflowTemplate is a template to be ran as part of a workflow @@ -33,8 +27,3 @@ type Matcher struct { // Subtemplates are ran if the name of matcher matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` } - -// GetPath of the workflow -func (w *Workflow) GetPath() string { - return w.path -} From 088c8770cc03c373f5853d213bc11f43f96ced8c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 16:33:25 +0530 Subject: [PATCH 36/92] Compiling templates + misc stuff --- v2/internal/runner/runner.go | 21 +++++++++------------ v2/pkg/templates/compile.go | 25 ++++++++++++++++++------- v2/pkg/templates/templates.go | 2 ++ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index d7e8ddd7b..368616ac5 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -239,22 +239,19 @@ func (r *Runner) RunEnumeration() { for _, t := range availableTemplates { wgtemplates.Add() - go func(template interface{}) { - defer wgtemplates.Done() - switch tt := template.(type) { - case *templates.Template: - for _, request := range tt.RequestsDNS { - results.Or(r.processTemplateWithList(p, tt, request)) - } - for _, request := range tt.BulkRequestsHTTP { - results.Or(r.processTemplateWithList(p, tt, request)) - } - case *workflows.Workflow: + go func(template *templates.Template) { + if template.Workflow != nil { results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow))) + + } + for _, request := range template.RequestsDNS { + results.Or(r.processTemplateWithList(p, tt, request)) + } + for _, request := range template.RequestsHTTP { + results.Or(r.processTemplateWithList(p, tt, request)) } }(t) } - wgtemplates.Wait() p.Stop() } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 4fc5d0bf6..be7e24df3 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -6,6 +6,8 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "gopkg.in/yaml.v2" ) @@ -30,23 +32,32 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { options.TemplateInfo = template.Info options.TemplatePath = file + // We don't support both http and dns in a single template + if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 { + return nil, fmt.Errorf("both http and dns requests for %s", template.ID) + } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) <= 0 { + if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } // Compile the requests found for _, request := range template.RequestsDNS { - if err := request.Compile(options); err != nil { - return nil, errors.Wrap(err, "could not compile dns request") - } template.totalRequests += request.Requests() } for _, request := range template.RequestsHTTP { - if err := request.Compile(options); err != nil { - return nil, errors.Wrap(err, "could not compile dns request") - } template.totalRequests += request.Requests() } + if len(template.RequestsDNS) > 0 { + template.executer = dns.NewExecuter(template.RequestsDNS, options) + err = template.executer.Compile() + } + if len(template.RequestsHTTP) > 0 { + template.executer = http.NewExecuter(template.RequestsHTTP, options) + err = template.executer.Compile() + } + if err != nil { + return nil, errors.Wrap(err, "could not compile request") + } return template, nil } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 41b7ff018..bbd679db4 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -1,6 +1,7 @@ package templates import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" @@ -22,6 +23,7 @@ type Template struct { path string totalRequests int + executer protocols.Executer } // GetPath returns the path of the template. From c42536f5e88f83f5bd6d1ee837e1ba9ca8273cf7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 18:02:45 +0530 Subject: [PATCH 37/92] Added catalogue + template-workflow running + misc --- v2/cmd/nuclei/main.go | 2 +- v2/go.mod | 13 +- v2/go.sum | 35 +-- v2/internal/runner/options.go | 6 +- v2/internal/runner/paths.go | 44 ---- v2/internal/runner/processor.go | 314 ++----------------------- v2/internal/runner/runner.go | 27 +-- v2/internal/runner/templates.go | 114 +-------- v2/pkg/catalogue/catalogue.go | 14 ++ v2/pkg/catalogue/find.go | 153 ++++++++++++ v2/pkg/catalogue/ignore.go | 70 ++++++ v2/pkg/catalogue/path.go | 45 ++++ v2/pkg/protocols/http/build_request.go | 9 +- v2/pkg/protocols/http/executer.go | 2 +- v2/pkg/protocols/http/request.go | 8 +- v2/pkg/protocols/protocols.go | 3 + v2/pkg/templates/compile.go | 79 ++++++- v2/pkg/templates/templates.go | 19 +- v2/pkg/workflows/execute.go | 10 +- v2/pkg/workflows/workflows.go | 6 +- 20 files changed, 454 insertions(+), 519 deletions(-) delete mode 100644 v2/internal/runner/paths.go create mode 100644 v2/pkg/catalogue/catalogue.go create mode 100644 v2/pkg/catalogue/find.go create mode 100644 v2/pkg/catalogue/ignore.go create mode 100644 v2/pkg/catalogue/path.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 3fad543dc..66333e821 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -11,7 +11,7 @@ func main() { nucleiRunner, err := runner.New(options) if err != nil { - gologger.Fatalf("Could not create runner: %s\n", err) + gologger.Fatal().Msgf("Could not create runner: %s\n", err) } nucleiRunner.RunEnumeration() diff --git a/v2/go.mod b/v2/go.mod index c3304b641..4b25d93ff 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,27 +6,32 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/corpix/uarand v0.1.1 - github.com/d5/tengo/v2 v2.6.2 github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 + github.com/kr/pretty v0.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/miekg/dns v1.1.35 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.7 github.com/projectdiscovery/collaborator v0.0.2 github.com/projectdiscovery/fastdialer v0.0.2 - github.com/projectdiscovery/gologger v1.0.1 + github.com/projectdiscovery/gologger v1.1.3 github.com/projectdiscovery/hmap v0.0.1 - github.com/projectdiscovery/nuclei/v2 v2.2.0 github.com/projectdiscovery/rawhttp v0.0.4 github.com/projectdiscovery/retryabledns v1.0.5 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/segmentio/ksuid v1.0.3 github.com/spaolacci/murmur3 v1.1.0 + github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.6.1 + go.uber.org/atomic v1.7.0 + go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.1.0 golang.org/x/net v0.0.0-20201216054612-986b41b23924 + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index 83a7183d2..20b07aa3c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -8,8 +8,6 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= -github.com/d5/tengo/v2 v2.6.2 h1:AnPhA/Y5qrNLb5QSWHU9uXq25T3QTTdd2waTgsAHMdc= -github.com/d5/tengo/v2 v2.6.2/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,6 +32,11 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -43,8 +46,12 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= @@ -55,21 +62,16 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/clistats v0.0.5/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/clistats v0.0.7 h1:Q/erjrk2p3BIQq1RaHVtBpgboghNz0u1/lyQ2fr8Cn0= github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= -github.com/projectdiscovery/collaborator v0.0.1/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s= github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.1/go.mod h1:d24GUzSb93wOY7lu4gJmXAzfomqAGEcRrInEVrM6zbc= github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13RN+TmcybEvbRc= github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA= -github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= -github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= +github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU= +github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/nuclei/v2 v2.2.0 h1:nUrTXM/AIJ8PfEPxEl/pkAHj7iu0TgAkE3e075a1JN0= -github.com/projectdiscovery/nuclei/v2 v2.2.0/go.mod h1:JIgYr5seElQh161hT/BUw3g1C4UuWR+VAcT16aZdyJ8= github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E= github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI= @@ -78,11 +80,12 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= -github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= @@ -91,11 +94,12 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -105,18 +109,18 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -131,13 +135,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 5186db62e..4134cb6b1 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -5,6 +5,7 @@ import ( "flag" "net/url" "os" + "path" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -16,6 +17,9 @@ import ( func ParseOptions() *types.Options { options := &types.Options{} + home, _ := os.UserHomeDir() + templatesDirectory := path.Join(home, "nuclei-templates") + flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") @@ -39,7 +43,7 @@ func ParseOptions() *types.Options { flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") - flag.StringVar(&options.TemplatesDirectory, "update-directory", "", "Directory to use for storing nuclei-templates") + flag.StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates") flag.BoolVar(&options.JSON, "json", false, "Write json output to files") flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output") flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") diff --git a/v2/internal/runner/paths.go b/v2/internal/runner/paths.go deleted file mode 100644 index be4b42a33..000000000 --- a/v2/internal/runner/paths.go +++ /dev/null @@ -1,44 +0,0 @@ -package runner - -import ( - "fmt" - "os" - "path" - "strings" - - "github.com/projectdiscovery/gologger" -) - -// isRelative checks if a given path is a relative path -func isRelative(filePath string) bool { - if strings.HasPrefix(filePath, "/") || strings.Contains(filePath, ":\\") { - return false - } - return true -} - -// resolvePath gets the absolute path to the template by either -// looking in the current directory or checking the nuclei templates directory. -// -// Current directory is given preference over the nuclei-templates directory. -func (r *Runner) resolvePath(templateName string) (string, error) { - curDirectory, err := os.Getwd() - if err != nil { - return "", err - } - - templatePath := path.Join(curDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debug().Msgf("Found template in current directory: %s\n", templatePath) - return templatePath, nil - } - - if r.templatesConfig != nil { - templatePath := path.Join(r.templatesConfig.TemplatesDirectory, templateName) - if _, err := os.Stat(templatePath); !os.IsNotExist(err) { - gologger.Debug().Msgf("Found template in nuclei-templates directory: %s\n", templatePath) - return templatePath, nil - } - } - return "", fmt.Errorf("no such path found: %s", templateName) -} diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 82fbb46f7..c4bfe2bfa 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -6,319 +6,57 @@ import ( "path" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" ) -/* -// workflowTemplates contains the initialized workflow templates per template group -type workflowTemplates struct { - Name string - Templates []*workflows.Template -} - -// processTemplateWithList processes a template and runs the enumeration on all the targets -func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { - var httpExecuter *executer.HTTPExecuter - var dnsExecuter *executer.DNSExecuter - var err error - - // Create an executer based on the request type. - switch value := request.(type) { - case *requests.DNSRequest: - dnsExecuter = executer.NewDNSExecuter(&executer.DNSOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: template, - DNSRequest: value, - Writer: r.output, - VHost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - NoMeta: r.options.NoMeta, - ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, - Decolorizer: r.decolorizer, - RateLimiter: r.ratelimiter, - }) - case *requests.BulkHTTPRequest: - httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: template, - BulkHTTPRequest: value, - Writer: r.output, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - JSON: r.options.JSON, - Vhost: r.options.Vhost, - JSONRequests: r.options.JSONRequests, - NoMeta: r.options.NoMeta, - CookieReuse: value.CookieReuse, - ColoredOutput: !r.options.NoColor, - Colorizer: &r.colorizer, - Decolorizer: r.decolorizer, - StopAtFirstMatch: r.options.StopAtFirstMatch, - PF: r.pf, - Dialer: r.dialer, - RateLimiter: r.ratelimiter, - }) - } - - if err != nil { - p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount()) - gologger.Warningf("Could not create http client: %s\n", err) - - return false - } - - var globalresult atomicboolean.AtomBool - +// processTemplateWithList process a template on the URL list +func (r *Runner) processTemplateWithList(template *templates.Template) bool { + results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { + r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) wg.Add() go func(URL string) { defer wg.Done() - - var result *executer.Result - - if httpExecuter != nil { - result = httpExecuter.ExecuteHTTP(p, URL) - globalresult.Or(result.GotResults) - } - - if dnsExecuter != nil { - result = dnsExecuter.ExecuteDNS(p, URL) - globalresult.Or(result.GotResults) - } - - if result.Error != nil { - gologger.Warningf("[%s] Could not execute step: %s\n", r.colorizer.Colorizer.BrightBlue(template.ID), result.Error) + match, err := template.Executer.Execute(URL) + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } else { + results.CAS(false, match) } }(URL) return nil }) - wg.Wait() - - // See if we got any results from the executers - return globalresult.Get() + return results.Load() } -// ProcessWorkflowWithList coming from stdin or list of targets -func (r *Runner) processWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) bool { - result := false - - workflowTemplatesList, err := r.preloadWorkflowTemplates(p, workflow) - if err != nil { - gologger.Warningf("Could not preload templates for workflow %s: %s\n", workflow.ID, err) - return false - } - logicBytes := []byte(workflow.Logic) - +// processTemplateWithList process a template on the URL list +func (r *Runner) processWorkflowWithList(template *templates.Template) bool { + results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hm.Scan(func(k, _ []byte) error { - targetURL := string(k) + + r.hostMap.Scan(func(k, _ []byte) error { + URL := string(k) wg.Add() - - go func(targetURL string) { + go func(URL string) { defer wg.Done() - - script := tengo.NewScript(logicBytes) - if !r.options.Sandbox { - script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) - } else { - script.SetImports(stdlib.GetModuleMap(sandboxedModules...)) - } - - variables := make(map[string]*workflows.NucleiVar) - for _, workflowTemplate := range *workflowTemplatesList { - name := workflowTemplate.Name - variable := &workflows.NucleiVar{Templates: workflowTemplate.Templates, URL: targetURL} - err := script.Add(name, variable) - if err != nil { - gologger.Errorf("Could not initialize script for workflow '%s': %s\n", workflow.ID, err) - continue - } - variables[name] = variable - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.options.MaxWorkflowDuration)*time.Minute) - defer cancel() - - _, err := script.RunContext(ctx) + match, err := template.RunWorkflow(URL) if err != nil { - gologger.Errorf("Could not execute workflow '%s': %s\n", workflow.ID, err) + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } else { + results.CAS(false, match) } - - for _, variable := range variables { - result = !variable.IsFalsy() - if result { - break - } - } - }(targetURL) + }(URL) return nil }) - wg.Wait() - - return result + return results.Load() } -func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workflows.Workflow) (*[]workflowTemplates, error) { - var jar *cookiejar.Jar - - if workflow.CookieReuse { - var err error - jar, err = cookiejar.New(nil) - if err != nil { - return nil, err - } - } - - // Single yaml provided - var wflTemplatesList []workflowTemplates - - for name, value := range workflow.Variables { - // Check if the template is an absolute path or relative path. - // If the path is absolute, use it. Otherwise, - if isRelative(value) { - newPath, err := r.resolvePath(value) - if err != nil { - newPath, err = resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value) - if err != nil { - return nil, err - } - } - - value = newPath - } - - var wtlst []*workflows.Template - - if strings.HasSuffix(value, ".yaml") { - t, err := templates.Parse(value) - if err != nil { - return nil, err - } - - template := &workflows.Template{Progress: p} - if len(t.BulkRequestsHTTP) > 0 { - template.HTTPOptions = &executer.HTTPOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Writer: r.output, - Template: t, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - Vhost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - CookieJar: jar, - ColoredOutput: !r.options.NoColor, - Colorizer: &r.colorizer, - Decolorizer: r.decolorizer, - PF: r.pf, - RateLimiter: r.ratelimiter, - NoMeta: r.options.NoMeta, - StopAtFirstMatch: r.options.StopAtFirstMatch, - Dialer: r.dialer, - } - } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executer.DNSOptions{ - TraceLog: r.traceLog, - Debug: r.options.Debug, - Template: t, - Writer: r.output, - VHost: r.options.Vhost, - JSON: r.options.JSON, - JSONRequests: r.options.JSONRequests, - ColoredOutput: !r.options.NoColor, - Colorizer: r.colorizer, - Decolorizer: r.decolorizer, - NoMeta: r.options.NoMeta, - RateLimiter: r.ratelimiter, - } - } - - if template.DNSOptions != nil || template.HTTPOptions != nil { - wtlst = append(wtlst, template) - } - } else { - matches := []string{} - - err := godirwalk.Walk(value, &godirwalk.Options{ - Callback: func(path string, d *godirwalk.Dirent) error { - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - matches = append(matches, path) - } - - return nil - }, - ErrorCallback: func(path string, err error) godirwalk.ErrorAction { - return godirwalk.SkipNode - }, - Unsorted: true, - }) - - if err != nil { - return nil, err - } - - // 0 matches means no templates were found in directory - if len(matches) == 0 { - return nil, fmt.Errorf("no match found in the directory %s", value) - } - - for _, match := range matches { - t, err := templates.Parse(match) - if err != nil { - return nil, err - } - template := &workflows.Template{Progress: p} - if len(t.BulkRequestsHTTP) > 0 { - template.HTTPOptions = &executer.HTTPOptions{ - Debug: r.options.Debug, - Writer: r.output, - Template: t, - Timeout: r.options.Timeout, - Retries: r.options.Retries, - ProxyURL: r.options.ProxyURL, - ProxySocksURL: r.options.ProxySocksURL, - RandomAgent: r.options.RandomAgent, - CustomHeaders: r.options.CustomHeaders, - Vhost: r.options.Vhost, - CookieJar: jar, - TraceLog: r.traceLog, - } - } else if len(t.RequestsDNS) > 0 { - template.DNSOptions = &executer.DNSOptions{ - Debug: r.options.Debug, - Template: t, - Writer: r.output, - VHost: r.options.Vhost, - TraceLog: r.traceLog, - } - } - if template.DNSOptions != nil || template.HTTPOptions != nil { - wtlst = append(wtlst, template) - } - } - } - wflTemplatesList = append(wflTemplatesList, workflowTemplates{Name: name, Templates: wtlst}) - } - return &wflTemplatesList, nil -}*/ - // resolvePathWithBaseFolder resolves a path with the base folder func resolvePathWithBaseFolder(baseFolder, templateName string) (string, error) { templatePath := path.Join(baseFolder, templateName) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 368616ac5..3b6b5889d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,13 +11,13 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/collaborator" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/internal/progress" - "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/remeh/sizedwaitgroup" + "go.uber.org/atomic" "go.uber.org/ratelimit" ) @@ -29,6 +29,7 @@ type Runner struct { templatesConfig *nucleiConfig options *types.Options projectFile *projectfile.ProjectFile + catalogue *catalogue.Catalogue progress *progress.Progress colorizer aurora.Aurora severityColors *colorizer.Colorizer @@ -61,6 +62,7 @@ func New(options *types.Options) (*Runner, error) { if runner.templatesConfig != nil { runner.readNucleiIgnoreFile() } + runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil { gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err) @@ -176,8 +178,8 @@ func (r *Runner) Close() { // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { // resolves input templates definitions and any optional exclusion - includedTemplates := r.getTemplatesFor(r.options.Templates) - excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates) + includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates) + excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates) // defaults to all templates allTemplates := includedTemplates @@ -222,10 +224,10 @@ func (r *Runner) RunEnumeration() { if t.Workflow != nil { continue } - totalRequests += int64(t.Requests()) * r.inputCount + totalRequests += int64(t.TotalRequests) * r.inputCount } - results := atomicboolean.New() + results := &atomic.Bool{} wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) // Starts polling or ignore collaborator.DefaultCollaborator.Poll() @@ -241,14 +243,9 @@ func (r *Runner) RunEnumeration() { wgtemplates.Add() go func(template *templates.Template) { if template.Workflow != nil { - results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow))) - - } - for _, request := range template.RequestsDNS { - results.Or(r.processTemplateWithList(p, tt, request)) - } - for _, request := range template.RequestsHTTP { - results.Or(r.processTemplateWithList(p, tt, request)) + results.CAS(false, r.processWorkflowWithList(template)) + } else { + results.CAS(false, r.processTemplateWithList(template)) } }(t) } @@ -256,7 +253,7 @@ func (r *Runner) RunEnumeration() { p.Stop() } - if !results.Get() { + if !results.Load() { if r.output != nil { r.output.Close() os.Remove(r.options.Output) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index b8a11d8f1..1497c1287 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -3,7 +3,6 @@ package runner import ( "fmt" "os" - "path/filepath" "strings" "github.com/karrick/godirwalk" @@ -12,105 +11,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates" ) -// getTemplatesFor parses the specified input template definitions and returns a list of unique, absolute template paths. -func (r *Runner) getTemplatesFor(definitions []string) []string { - // keeps track of processed dirs and files - processed := make(map[string]bool) - allTemplates := []string{} - - // parses user input, handle file/directory cases and produce a list of unique templates - for _, t := range definitions { - var absPath string - var err error - - if strings.Contains(t, "*") { - dirs := strings.Split(t, "/") - priorDir := strings.Join(dirs[:len(dirs)-1], "/") - absPath, err = r.resolvePathIfRelative(priorDir) - absPath += "/" + dirs[len(dirs)-1] - } else { - // resolve and convert relative to absolute path - absPath, err = r.resolvePathIfRelative(t) - } - if err != nil { - gologger.Error().Msgf("Could not find template file '%s': %s\n", t, err) - continue - } - - // Template input includes a wildcard - if strings.Contains(absPath, "*") { - var matches []string - matches, err = filepath.Glob(absPath) - if err != nil { - gologger.Error().Msgf("Wildcard found, but unable to glob '%s': %s\n", absPath, err) - continue - } - - // couldn't find templates in directory - if len(matches) == 0 { - gologger.Error().Msgf("Error, no templates were found with '%s'.\n", absPath) - continue - } else { - gologger.Verbose().Msgf("Identified %d templates\n", len(matches)) - } - - for _, match := range matches { - if !r.checkIfInNucleiIgnore(match) { - processed[match] = true - allTemplates = append(allTemplates, match) - } - } - } else { - // determine file/directory - isFile, err := isFilePath(absPath) - if err != nil { - gologger.Error().Msgf("Could not stat '%s': %s\n", absPath, err) - continue - } - // test for uniqueness - if !isNewPath(absPath, processed) { - continue - } - // mark this absolute path as processed - // - if it's a file, we'll never process it again - // - if it's a dir, we'll never walk it again - processed[absPath] = true - - if isFile { - allTemplates = append(allTemplates, absPath) - } else { - matches := []string{} - - // Recursively walk down the Templates directory and run all the template file checks - err := directoryWalker(absPath, - func(path string, d *godirwalk.Dirent) error { - if !d.IsDir() && strings.HasSuffix(path, ".yaml") { - if !r.checkIfInNucleiIgnore(path) && isNewPath(path, processed) { - matches = append(matches, path) - processed[path] = true - } - } - return nil - }, - ) - // directory couldn't be walked - if err != nil { - gologger.Error().Msgf("Could not find templates in directory '%s': %s\n", absPath, err) - continue - } - - // couldn't find templates in directory - if len(matches) == 0 { - gologger.Error().Msgf("Error, no templates were found in '%s'.\n", absPath) - continue - } - allTemplates = append(allTemplates, matches...) - } - } - } - return allTemplates -} - // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []*templates.Template, workflowCount int) { @@ -148,6 +48,7 @@ func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { Output: r.output, Options: r.options, Progress: r.progress, + Catalogue: r.catalogue, RateLimiter: r.ratelimiter, ProjectFile: r.projectFile, } @@ -211,19 +112,6 @@ func (r *Runner) listAvailableTemplates() { } } -func (r *Runner) resolvePathIfRelative(filePath string) (string, error) { - if isRelative(filePath) { - newPath, err := r.resolvePath(filePath) - - if err != nil { - return "", err - } - return newPath, nil - } - - return filePath, nil -} - func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool { for _, s := range allowedSeverities { if s != "" && strings.HasPrefix(templateSeverity, s) { diff --git a/v2/pkg/catalogue/catalogue.go b/v2/pkg/catalogue/catalogue.go new file mode 100644 index 000000000..eaba68593 --- /dev/null +++ b/v2/pkg/catalogue/catalogue.go @@ -0,0 +1,14 @@ +package catalogue + +// Catalogue is a template catalouge helper implementation +type Catalogue struct { + ignoreFiles []string + templatesDirectory string +} + +// New creates a new catalogue structure using provided input items +func New(directory string) *Catalogue { + catalogue := &Catalogue{templatesDirectory: directory} + catalogue.readNucleiIgnoreFile() + return catalogue +} diff --git a/v2/pkg/catalogue/find.go b/v2/pkg/catalogue/find.go new file mode 100644 index 000000000..0191f7873 --- /dev/null +++ b/v2/pkg/catalogue/find.go @@ -0,0 +1,153 @@ +package catalogue + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" +) + +// GetTemplatesPath returns a list of absolute paths for the provided template list. +func (c *Catalogue) GetTemplatesPath(definitions []string) []string { + // keeps track of processed dirs and files + processed := make(map[string]bool) + allTemplates := []string{} + + for _, t := range definitions { + paths, err := c.GetTemplatePath(t) + if err != nil { + gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) + } + for _, path := range paths { + if _, ok := processed[path]; !ok { + processed[path] = true + allTemplates = append(allTemplates, path) + } + } + } + gologger.Verbose().Msgf("Identified %d templates", len(allTemplates)) + return allTemplates +} + +// GetTemplatePath parses the specified input template path and returns a compiled +// list of finished absolute paths to the templates evaluating any glob patterns +// or folders provided as in. +func (c *Catalogue) GetTemplatePath(target string) ([]string, error) { + processed := make(map[string]struct{}) + + absPath, err := c.convertPathToAbsolute(target) + if err != nil { + return nil, errors.Wrapf(err, "could not find template file") + } + + // Template input includes a wildcard + if strings.Contains(absPath, "*") { + matches, err := c.findGlobPathMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find glob matches") + } + if len(matches) == 0 { + return nil, errors.Errorf("no templates found for path") + } + return matches, nil + } + + // Template input is either a file or a directory + match, file, err := c.findFileMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find file") + } + if file { + if match != "" { + return []string{match}, nil + } + return nil, nil + } + + // Recursively walk down the Templates directory and run all + // the template file checks + matches, err := c.findDirectoryMatches(absPath, processed) + if err != nil { + return nil, errors.Wrap(err, "could not find directory matches") + } + if len(matches) == 0 { + return nil, errors.Errorf("no templates found in path") + } + return matches, nil +} + +// convertPathToAbsolute resolves the paths provided to absolute paths +// before doing any operations on them regardless of them being blob, folders, files, etc. +func (c *Catalogue) convertPathToAbsolute(t string) (string, error) { + if strings.Contains(t, "*") { + file := path.Base(t) + absPath, err := c.ResolvePath(path.Dir(t), "") + if err != nil { + return "", err + } + return path.Join(absPath, file), nil + } + return c.ResolvePath(t, "") +} + +// findGlobPathMatches returns the matched files from a glob path +func (c *Catalogue) findGlobPathMatches(absPath string, processed map[string]struct{}) ([]string, error) { + matches, err := filepath.Glob(absPath) + if err != nil { + return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err) + } + results := make([]string, 0, len(matches)) + for _, match := range matches { + if _, ok := processed[match]; !ok { + processed[match] = struct{}{} + results = append(results, match) + } + } + return results, nil +} + +// findFileMatches finds if a path is an absolute file. If the path +// is a file, it returns true otherwise false with no errors. +func (c *Catalogue) findFileMatches(absPath string, processed map[string]struct{}) (string, bool, error) { + info, err := os.Stat(absPath) + if err != nil { + return "", false, err + } + if !info.Mode().IsRegular() { + return "", false, nil + } + if _, ok := processed[absPath]; !ok { + processed[absPath] = struct{}{} + return absPath, true, nil + } + return "", true, nil +} + +// findDirectoryMatches finds matches for templates from a directory +func (c *Catalogue) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) { + var results []string + err := godirwalk.Walk(absPath, &godirwalk.Options{ + Unsorted: true, + ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { + return godirwalk.SkipNode + }, + Callback: func(path string, d *godirwalk.Dirent) error { + if !d.IsDir() && strings.HasSuffix(path, ".yaml") { + if c.checkIfInNucleiIgnore(path) { + return nil + } + + if _, ok := processed[path]; !ok { + results = append(results, path) + processed[path] = struct{}{} + } + } + return nil + }, + }) + return results, err +} diff --git a/v2/pkg/catalogue/ignore.go b/v2/pkg/catalogue/ignore.go new file mode 100644 index 000000000..591d03b44 --- /dev/null +++ b/v2/pkg/catalogue/ignore.go @@ -0,0 +1,70 @@ +package catalogue + +import ( + "bufio" + "os" + "path" + "strings" +) + +const nucleiIgnoreFile = ".nuclei-ignore" + +// readNucleiIgnoreFile reads the nuclei ignore file marking it in map +func (c *Catalogue) readNucleiIgnoreFile() { + file, err := os.Open(path.Join(c.templatesDirectory, nucleiIgnoreFile)) + if err != nil { + return + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + if strings.HasPrefix(text, "#") { + continue + } + c.ignoreFiles = append(c.ignoreFiles, text) + } +} + +// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. +func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { + if c.templatesDirectory == "" { + return false + } + + for _, paths := range c.ignoreFiles { + // If we have a path to ignore, check if it's in the item. + if paths[len(paths)-1] == '/' { + if strings.Contains(item, paths) { + return true + } + + continue + } + // Check for file based extension in ignores + if strings.HasSuffix(item, paths) { + return true + } + } + return false +} + +// ignoreFilesWithExcludes ignores results with exclude paths +func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string { + excludeMap := make(map[string]struct{}, len(excluded)) + for _, excl := range excluded { + excludeMap[excl] = struct{}{} + } + + templates := make([]string, 0, len(results)) + for _, incl := range results { + if _, found := excludeMap[incl]; !found { + templates = append(templates, incl) + } + } + return templates +} diff --git a/v2/pkg/catalogue/path.go b/v2/pkg/catalogue/path.go new file mode 100644 index 000000000..2ff00d0e7 --- /dev/null +++ b/v2/pkg/catalogue/path.go @@ -0,0 +1,45 @@ +package catalogue + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +// ResolvePath resolves the path to an absolute one in various ways. +// +// It checks if the filename is an absolute path, looks in the current directory +// or checking the nuclei templates directory. If a second path is given, +// it also tries to find paths relative to that second path. +func (c *Catalogue) ResolvePath(templateName, second string) (string, error) { + if strings.HasPrefix(templateName, "/") || strings.Contains(templateName, ":\\") { + return templateName, nil + } + + if second != "" { + secondBasePath := path.Join(filepath.Dir(second), templateName) + if _, err := os.Stat(secondBasePath); !os.IsNotExist(err) { + return secondBasePath, nil + } + } + + curDirectory, err := os.Getwd() + if err != nil { + return "", err + } + + templatePath := path.Join(curDirectory, templateName) + if _, err := os.Stat(templatePath); !os.IsNotExist(err) { + return templatePath, nil + } + + if c.templatesDirectory != "" { + templatePath := path.Join(c.templatesDirectory, templateName) + if _, err := os.Stat(templatePath); !os.IsNotExist(err) { + return templatePath, nil + } + } + return "", fmt.Errorf("no such path found: %s", templateName) +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 49794c52b..23e79d2eb 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -130,13 +130,12 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values) } -// Remaining returns the remaining number of requests for the generator -func (r *requestGenerator) Remaining() int { +// Total returns the total number of requests for the generator +func (r *requestGenerator) Total() int { if r.payloadIterator != nil { - payloadRemaining := r.payloadIterator.Remaining() - return (len(r.request.Raw) - r.currentIndex + 1) * payloadRemaining + return len(r.request.Raw) * r.payloadIterator.Remaining() } - return len(r.request.Path) - r.currentIndex + 1 + return len(r.request.Path) } // baseURLWithTemplatePrefs returns the url for BaseURL keeping diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go index fdcdb7586..df92c1c34 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/http/executer.go @@ -33,7 +33,7 @@ func (e *Executer) Compile() error { func (e *Executer) Requests() int { var count int for _, request := range e.requests { - count += int64(request.Requests()) + count += int(request.Requests()) } return count } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 89dcb74da..2edec90e0 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -75,7 +75,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } swg.Add() @@ -136,7 +136,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } request.pipelinedClient = pipeclient @@ -187,7 +187,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } @@ -201,7 +201,7 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int e.options.Progress.IncrementRequests() if request.original.options.Options.StopAtFirstMatch && len(output) > 0 { - e.options.Progress.DecrementRequests(int64(generator.Remaining())) + e.options.Progress.DecrementRequests(int64(generator.Total())) break } } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 67ea9e21d..3293fc8e1 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -2,6 +2,7 @@ package protocols import ( "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -38,6 +39,8 @@ type ExecuterOptions struct { Progress *progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. RateLimiter ratelimit.Limiter + // Catalogue is a template catalogue implementation for nuclei + Catalogue *catalogue.Catalogue // ProjectFile is the project file for nuclei ProjectFile *projectfile.ProjectFile } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index be7e24df3..576608921 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -27,7 +28,6 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { defer f.Close() // Setting up variables regarding template metadata - template.path = file options.TemplateID = template.ID options.TemplateInfo = template.Info options.TemplatePath = file @@ -37,27 +37,90 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { return nil, fmt.Errorf("both http and dns requests for %s", template.ID) } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflow.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } + // Compile the workflow request + if template.Workflow != nil { + if err := template.compileWorkflow(options); err != nil { + return nil, errors.Wrap(err, "could not compile workflow") + } + } + // Compile the requests found for _, request := range template.RequestsDNS { - template.totalRequests += request.Requests() + template.TotalRequests += request.Requests() } for _, request := range template.RequestsHTTP { - template.totalRequests += request.Requests() + template.TotalRequests += request.Requests() } if len(template.RequestsDNS) > 0 { - template.executer = dns.NewExecuter(template.RequestsDNS, options) - err = template.executer.Compile() + template.Executer = dns.NewExecuter(template.RequestsDNS, options) + err = template.Executer.Compile() } if len(template.RequestsHTTP) > 0 { - template.executer = http.NewExecuter(template.RequestsHTTP, options) - err = template.executer.Compile() + template.Executer = http.NewExecuter(template.RequestsHTTP, options) + err = template.Executer.Compile() } if err != nil { return nil, errors.Wrap(err, "could not compile request") } return template, nil } + +// compileWorkflow compiles the workflow for execution +func (t *Template) compileWorkflow(options *protocols.ExecuterOptions) error { + for _, workflow := range t.Workflows { + if err := t.parseWorkflow(workflow, options); err != nil { + return err + } + } + return nil +} + +// parseWorkflow parses and compiles all templates in a workflow recursively +func (t *Template) parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error { + if err := t.parseWorkflowTemplate(workflow, options); err != nil { + return err + } + for _, subtemplates := range workflow.Subtemplates { + if err := t.parseWorkflow(subtemplates, options); err != nil { + return err + } + } + for _, matcher := range workflow.Matchers { + for _, subtemplates := range matcher.Subtemplates { + if err := t.parseWorkflow(subtemplates, options); err != nil { + return err + } + } + } + return nil +} + +// parseWorkflowTemplate parses a workflow template creating an executer +func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions) error { + opts := protocols.ExecuterOptions{ + Output: options.Output, + Options: options.Options, + Progress: options.Progress, + Catalogue: options.Catalogue, + RateLimiter: options.RateLimiter, + ProjectFile: options.ProjectFile, + } + paths, err := options.Catalogue.GetTemplatePath(workflow.Template) + if err != nil { + return errors.Wrap(err, "could not get workflow template") + } + if len(paths) != 1 { + return errors.Wrap(err, "invalid number of templates matched") + } + + template, err := Parse(paths[0], &opts) + if err != nil { + return errors.Wrap(err, "could not parse workflow template") + } + workflow.Executer = template.Executer + return nil +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index bbd679db4..d822c4ddd 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -17,21 +17,10 @@ type Template struct { RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` - // Workflows is a yaml based workflow declaration code. *workflows.Workflow - - path string - totalRequests int - executer protocols.Executer -} - -// GetPath returns the path of the template. -func (t *Template) GetPath() string { - return t.path -} - -// Requests returns the number of requests for the template -func (t *Template) Requests() int { - return t.totalRequests + // TotalRequests is the total number of requests for the template. + TotalRequests int + // Executer is the actual template executor for running template requests + Executer protocols.Executer } diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 5dc43e0ff..30f2feece 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -3,7 +3,7 @@ package workflows import "go.uber.org/atomic" // RunWorkflow runs a workflow on an input and returns true or false -func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) { +func (w *Workflow) RunWorkflow(input string) (bool, error) { results := &atomic.Bool{} for _, template := range w.Workflows { @@ -20,7 +20,9 @@ func (w *WorkflowTemplate) RunWorkflow(input string) (bool, error) { func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool) error { var firstMatched bool if len(template.Matchers) == 0 { - matched, err := template.executer.Execute(input) + w.options.Progress.AddToTotal(int64(template.Executer.Requests())) + + matched, err := template.Executer.Execute(input) if err != nil { return err } @@ -29,7 +31,9 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } if len(template.Matchers) > 0 { - output, err := template.executer.ExecuteWithResults(input) + w.options.Progress.AddToTotal(int64(template.Executer.Requests())) + + output, err := template.Executer.ExecuteWithResults(input) if err != nil { return err } diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index f69cdcaf1..73e752972 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -6,6 +6,8 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" type Workflow struct { // Workflows is a yaml based workflow declaration code. Workflows []*WorkflowTemplate `yaml:"workflows"` + + options *protocols.ExecuterOptions } // WorkflowTemplate is a template to be ran as part of a workflow @@ -16,8 +18,8 @@ type WorkflowTemplate struct { Matchers []*Matcher `yaml:"matchers"` // Subtemplates are ran if the template matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` - - executer protocols.Executer + // Executer performs the actual execution for the workflow template + Executer protocols.Executer } // Matcher performs conditional matching on the workflow template results. From 7933a9c70c1951a5e354e368e6769d20dd4d2af4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 18:15:27 +0530 Subject: [PATCH 38/92] Bug fixes --- v2/internal/runner/runner.go | 28 ++++++++++++++++++++++------ v2/pkg/templates/compile.go | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3b6b5889d..4dd7b1ec0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -14,6 +14,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -126,13 +128,11 @@ func New(options *types.Options) (*Runner, error) { } // Create the output file if asked - if options.Output != "" { - output, errWriter := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile) - if errWriter != nil { - gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, errWriter) - } - runner.output = output + output, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.JSON, options.Output, options.TraceLogFile) + if err != nil { + gologger.Fatal().Msgf("Could not create output file '%s': %s\n", options.Output, err) } + runner.output = output // Creates the progress tracking object var progressErr error @@ -177,6 +177,11 @@ func (r *Runner) Close() { // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { + err := r.initializeProtocols() + if err != nil { + gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) + } + // resolves input templates definitions and any optional exclusion includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates) excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates) @@ -261,3 +266,14 @@ func (r *Runner) RunEnumeration() { gologger.Info().Msgf("No results found. Happy hacking!") } } + +// initializeProtocols initializes all the protocols and their caches +func (r *Runner) initializeProtocols() error { + if err := dnsclientpool.Init(r.options); err != nil { + return err + } + if err := httpclientpool.Init(r.options); err != nil { + return err + } + return nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 576608921..e00c91308 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -37,7 +37,7 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { return nil, fmt.Errorf("both http and dns requests for %s", template.ID) } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsDNS)+len(template.Workflow.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP) == 0 && template.Workflow == nil { return nil, fmt.Errorf("no requests defined for %s", template.ID) } From 8bc95878811fa982646b5925f109a12cbff31006 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 18:48:13 +0530 Subject: [PATCH 39/92] Finished bugs, working state finally --- v2/go.mod | 1 + v2/go.sum | 20 ++++++++++++++++++++ v2/internal/runner/processor.go | 2 ++ v2/internal/runner/runner.go | 4 +++- v2/pkg/protocols/dns/dns.go | 2 +- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/templates/compile.go | 11 ++++++++--- v2/pkg/templates/templates.go | 2 +- v2/pkg/workflows/compile.go | 2 +- 9 files changed, 38 insertions(+), 8 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 4b25d93ff..e40840700 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,6 +6,7 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/blang/semver v3.5.1+incompatible github.com/corpix/uarand v0.1.1 + github.com/goccy/go-yaml v1.8.4 github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 diff --git a/v2/go.sum b/v2/go.sum index 20b07aa3c..0225d7a02 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -13,8 +13,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/goccy/go-yaml v1.8.4 h1:AOEdR7aQgbgwHznGe3BLkDQVujxCPUpHOZZcQcp8Y3M= +github.com/goccy/go-yaml v1.8.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -37,9 +45,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -87,6 +100,7 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -100,6 +114,7 @@ go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0ayQkEnKiOC1TtM3fWXFnoU= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -121,17 +136,22 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index c4bfe2bfa..7abe3920d 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -21,6 +21,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { wg.Add() go func(URL string) { defer wg.Done() + match, err := template.Executer.Execute(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) @@ -31,6 +32,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { return nil }) wg.Wait() + return results.Load() } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4dd7b1ec0..dbe9fd48a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -247,7 +247,9 @@ func (r *Runner) RunEnumeration() { for _, t := range availableTemplates { wgtemplates.Add() go func(template *templates.Template) { - if template.Workflow != nil { + defer wgtemplates.Done() + + if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) } else { results.CAS(false, r.processTemplateWithList(template)) diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 4512fe795..92385c3b6 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -28,7 +28,7 @@ type Request struct { Raw string `yaml:"raw,omitempty"` // Operators for the current request go here. - *operators.Operators + *operators.Operators `yaml:",inline"` // cache any variables that may be needed for operation. class uint16 diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 00cb2131b..c54ea92fb 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -58,7 +58,7 @@ type Request struct { Race bool `yaml:"race"` // Operators for the current request go here. - *operators.Operators + *operators.Operators `yaml:",omitempty,inline"` options *protocols.ExecuterOptions attackType generators.Type diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index e00c91308..91e5c8df4 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,12 +4,12 @@ import ( "fmt" "os" + "github.com/goccy/go-yaml" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" - "gopkg.in/yaml.v2" ) // Parse parses a yaml request template file @@ -37,12 +37,12 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { return nil, fmt.Errorf("both http and dns requests for %s", template.ID) } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP) == 0 && template.Workflow == nil { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } // Compile the workflow request - if template.Workflow != nil { + if len(template.Workflows) > 0 { if err := template.compileWorkflow(options); err != nil { return nil, errors.Wrap(err, "could not compile workflow") } @@ -66,6 +66,11 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { if err != nil { return nil, errors.Wrap(err, "could not compile request") } + if template.Executer != nil { + if err := template.Executer.Compile(); err != nil { + return nil, errors.Wrap(err, "could not compile template executer") + } + } return template, nil } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d822c4ddd..8e4adc365 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -18,7 +18,7 @@ type Template struct { // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` // Workflows is a yaml based workflow declaration code. - *workflows.Workflow + *workflows.Workflow `yaml:",omitempty,inline"` // TotalRequests is the total number of requests for the template. TotalRequests int // Executer is the actual template executor for running template requests diff --git a/v2/pkg/workflows/compile.go b/v2/pkg/workflows/compile.go index 3e973d8b7..2330b2f6d 100644 --- a/v2/pkg/workflows/compile.go +++ b/v2/pkg/workflows/compile.go @@ -3,8 +3,8 @@ package workflows import ( "os" + "github.com/goccy/go-yaml" "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) // Parse a yaml workflow file From 1bb4a2568af4b5571bdc898b6b8aba9ba8e55a22 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 19:35:16 +0530 Subject: [PATCH 40/92] Fixed panic in http executer --- v2/pkg/protocols/http/http.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index c54ea92fb..0d354a065 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -109,9 +109,12 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Requests returns the total number of requests the YAML rule will perform func (r *Request) Requests() int { - if len(r.Payloads) > 0 { + if r.generator != nil { payloadRequests := r.generator.NewIterator().Total() return len(r.Raw) * payloadRequests } + if len(r.Raw) > 0 { + return len(r.Raw) + } return len(r.Path) } From 4da3d31c729f9fb9d4b36902203963ade67c3d65 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 19:46:52 +0530 Subject: [PATCH 41/92] Fixed multiple bugs and panics --- v2/internal/runner/runner.go | 2 +- v2/internal/runner/templates.go | 7 +++---- v2/pkg/protocols/http/http.go | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index dbe9fd48a..f11fde039 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -226,7 +226,7 @@ func (r *Runner) RunEnumeration() { for _, t := range availableTemplates { // workflows will dynamically adjust the totals while running, as // it can't be know in advance which requests will be called - if t.Workflow != nil { + if len(t.Workflows) > 0 { continue } totalRequests += int64(t.TotalRequests) * r.inputCount diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 1497c1287..c37506a6b 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -27,13 +27,12 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string gologger.Error().Msgf("Could not parse file '%s': %s\n", match, err) continue } + if len(t.Workflows) > 0 { + workflowCount++ + } sev := strings.ToLower(t.Info["severity"]) if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) { parsedTemplates = append(parsedTemplates, t) - // Process the template like a workflow - if t.Workflow != nil { - workflowCount++ - } gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } else { gologger.Error().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 0d354a065..782e37066 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -80,6 +80,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { return errors.Wrap(err, "could not get dns client") } r.httpClient = client + r.options = options if len(r.Raw) > 0 { r.rawhttpClient = httpclientpool.GetRawHTTP() From fe7a5def2977ff9dcd6f4a546db14e0e068de3a4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 20:18:59 +0530 Subject: [PATCH 42/92] Fixed panic in workflows --- v2/pkg/templates/compile.go | 1 + v2/pkg/workflows/workflows.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 91e5c8df4..735500455 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -46,6 +46,7 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { if err := template.compileWorkflow(options); err != nil { return nil, errors.Wrap(err, "could not compile workflow") } + template.Workflow.Compile(options) } // Compile the requests found diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index 73e752972..b16615ed5 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -29,3 +29,8 @@ type Matcher struct { // Subtemplates are ran if the name of matcher matches. Subtemplates []*WorkflowTemplate `yaml:"subtemplates"` } + +// Compile compiles the workflow template for execution +func (w *Workflow) Compile(options *protocols.ExecuterOptions) { + w.options = options +} From 27ef27a51b669439b28bb573d577f4cbbbf46456 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 20:32:04 +0530 Subject: [PATCH 43/92] Fixed a bug with workflow output --- v2/internal/runner/processor.go | 6 ++---- v2/pkg/workflows/execute.go | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 7abe3920d..31136ed35 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -25,9 +25,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { match, err := template.Executer.Execute(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } else { - results.CAS(false, match) } + results.CAS(false, match) }(URL) return nil }) @@ -49,9 +48,8 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { match, err := template.RunWorkflow(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) - } else { - results.CAS(false, match) } + results.CAS(false, match) }(URL) return nil }) diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 30f2feece..9e74be30d 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -9,7 +9,7 @@ func (w *Workflow) RunWorkflow(input string) (bool, error) { for _, template := range w.Workflows { err := w.runWorkflowStep(template, input, results) if err != nil { - return false, err + return results.Load(), err } } return results.Load(), nil @@ -26,8 +26,10 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if err != nil { return err } - firstMatched = matched - results.CAS(false, matched) + if matched { + firstMatched = matched + results.CAS(false, matched) + } } if len(template.Matchers) > 0 { From 432693d7ff0d8171bf1c5030b17c3747a54aad38 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 29 Dec 2020 20:55:44 +0530 Subject: [PATCH 44/92] Misc --- v2/pkg/catalogue/find.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/catalogue/find.go b/v2/pkg/catalogue/find.go index 0191f7873..446ff430b 100644 --- a/v2/pkg/catalogue/find.go +++ b/v2/pkg/catalogue/find.go @@ -29,7 +29,9 @@ func (c *Catalogue) GetTemplatesPath(definitions []string) []string { } } } - gologger.Verbose().Msgf("Identified %d templates", len(allTemplates)) + if len(allTemplates) > 0 { + gologger.Verbose().Msgf("Identified %d templates", len(allTemplates)) + } return allTemplates } From ec57ac460f9560dd3d55285ae1dc5a7416a8ff8d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 13:26:55 +0530 Subject: [PATCH 45/92] Fixed a bug with conditions in nuclei --- v2/internal/runner/processor.go | 2 +- v2/pkg/operators/operators.go | 16 ++++++++++++---- v2/pkg/protocols/dns/dns.go | 9 ++++++--- v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/http/http.go | 9 ++++++--- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/templates/compile.go | 10 ++++++---- v2/pkg/templates/templates.go | 4 +++- v2/pkg/workflows/compile.go | 2 +- 9 files changed, 37 insertions(+), 19 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 31136ed35..fc9d6c9b2 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -45,7 +45,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { wg.Add() go func(URL string) { defer wg.Done() - match, err := template.RunWorkflow(URL) + match, err := template.CompiledWorkflow.RunWorkflow(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 16c2e8f4b..12fce8d1f 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -26,7 +26,7 @@ func (r *Operators) Compile() error { if r.MatchersCondition != "" { r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition] } else { - r.matchersCondition = matchers.ANDCondition + r.matchersCondition = matchers.ORCondition } for _, matcher := range r.Matchers { @@ -71,6 +71,7 @@ type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { matcherCondition := r.GetMatchersCondition() + var matches bool result := &Result{ Matches: make(map[string]struct{}), Extracts: make(map[string][]string), @@ -86,9 +87,10 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } else { // If the matcher has matched, and its an OR // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition { + if matcherCondition == matchers.ORCondition && matcher.Name != "" { result.Matches[matcher.Name] = struct{}{} } + matches = true } } @@ -108,12 +110,18 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac result.OutputExtracts = append(result.OutputExtracts, match) } } - result.Extracts[extractor.Name] = extractorResults + if len(extractorResults) > 0 { + result.Extracts[extractor.Name] = extractorResults + } } + // Don't print if we have matchers and they have not matched, irregardless of extractor + if len(r.Matchers) > 0 && !matches { + return nil, false + } // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. - if len(result.Extracts) > 0 || len(result.Matches) > 0 || matcherCondition == matchers.ANDCondition { + if len(result.Extracts) > 0 || matches { return result, true } return nil, false diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 92385c3b6..275d81266 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -28,7 +28,8 @@ type Request struct { Raw string `yaml:"raw,omitempty"` // Operators for the current request go here. - *operators.Operators `yaml:",inline"` + operators.Operators `yaml:",inline"` + CompiledOperators *operators.Operators // cache any variables that may be needed for operation. class uint16 @@ -48,10 +49,12 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.dnsClient = client - if r.Operators != nil { - if err := r.Operators.Compile(); err != nil { + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } + r.CompiledOperators = compiled } r.class = classToInt(r.Class) r.options = options diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 4e25fcf65..47779cf9f 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -55,7 +55,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} - if r.Operators != nil { + if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { return nil, nil diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 782e37066..40c8757bb 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -58,7 +58,8 @@ type Request struct { Race bool `yaml:"race"` // Operators for the current request go here. - *operators.Operators `yaml:",omitempty,inline"` + operators.Operators `yaml:",inline"` + CompiledOperators *operators.Operators options *protocols.ExecuterOptions attackType generators.Type @@ -85,10 +86,12 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if len(r.Raw) > 0 { r.rawhttpClient = httpclientpool.GetRawHTTP() } - if r.Operators != nil { - if err := r.Operators.Compile(); err != nil { + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } + r.CompiledOperators = compiled } if len(r.Payloads) > 0 { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 2edec90e0..cd3d562a1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -353,7 +353,7 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} - if e.Operators != nil { + if e.CompiledOperators != nil { result, ok := e.Operators.Execute(ouputEvent, e.Match, e.Extract) if !ok { return nil, nil diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 735500455..8f18df1dc 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,12 +4,12 @@ import ( "fmt" "os" - "github.com/goccy/go-yaml" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "gopkg.in/yaml.v2" ) // Parse parses a yaml request template file @@ -43,10 +43,12 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { // Compile the workflow request if len(template.Workflows) > 0 { - if err := template.compileWorkflow(options); err != nil { + compiled := &template.Workflow + if err := template.compileWorkflow(options, compiled); err != nil { return nil, errors.Wrap(err, "could not compile workflow") } template.Workflow.Compile(options) + template.CompiledWorkflow = compiled } // Compile the requests found @@ -76,8 +78,8 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { } // compileWorkflow compiles the workflow for execution -func (t *Template) compileWorkflow(options *protocols.ExecuterOptions) error { - for _, workflow := range t.Workflows { +func (t *Template) compileWorkflow(options *protocols.ExecuterOptions, workflows *workflows.Workflow) error { + for _, workflow := range workflows.Workflows { if err := t.parseWorkflow(workflow, options); err != nil { return err } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 8e4adc365..8b54d9f0b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -18,7 +18,9 @@ type Template struct { // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` // Workflows is a yaml based workflow declaration code. - *workflows.Workflow `yaml:",omitempty,inline"` + workflows.Workflow `yaml:",inline"` + CompiledWorkflow *workflows.Workflow + // TotalRequests is the total number of requests for the template. TotalRequests int // Executer is the actual template executor for running template requests diff --git a/v2/pkg/workflows/compile.go b/v2/pkg/workflows/compile.go index 2330b2f6d..3e973d8b7 100644 --- a/v2/pkg/workflows/compile.go +++ b/v2/pkg/workflows/compile.go @@ -3,8 +3,8 @@ package workflows import ( "os" - "github.com/goccy/go-yaml" "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) // Parse a yaml workflow file From a12051799d1d3cd9e4482a2b3ed333df3992a99f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 13:57:15 +0530 Subject: [PATCH 46/92] Added support for multiple sniper payloads --- .../protocols/common/generators/generators.go | 18 ++++++++++++------ .../common/generators/generators_test.go | 10 +++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 971cb3a8c..e3e4b5f9d 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -46,9 +46,6 @@ func New(payloads map[string]interface{}, Type Type, templatePath string) (*Gene generator.payloads = compiled // Validate the payload types - if Type == Sniper && len(compiled) > 1 { - return nil, errors.New("cannot use more than one payload set in sniper") - } if Type == PitchFork { var totalLength int for v := range compiled { @@ -104,7 +101,11 @@ func (i *Iterator) Remaining() int { func (i *Iterator) Total() int { count := 0 switch i.Type { - case Sniper, PitchFork: + case Sniper: + for _, p := range i.payloads { + count += len(p.values) + } + case PitchFork: count = len(i.payloads[0].values) case ClusterBomb: count = 1 @@ -133,9 +134,14 @@ func (i *Iterator) Value() (map[string]interface{}, bool) { func (i *Iterator) sniperValue() (map[string]interface{}, bool) { values := make(map[string]interface{}, 1) - payload := i.payloads[0] + currentIndex := i.msbIterator + payload := i.payloads[currentIndex] if !payload.next() { - return nil, false + i.msbIterator++ + if i.msbIterator == len(i.payloads) { + return nil, false + } + return i.sniperValue() } values[payload.name] = payload.value() payload.incrementPosition() diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 1380dac90..6dc29eabc 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -7,22 +7,22 @@ import ( ) func TestSniperGenerator(t *testing.T) { - usernames := []string{"admin", "password", "login", "test"} + usernames := []string{"admin", "password"} + moreUsernames := []string{"login", "test"} - generator, err := New(map[string]interface{}{"username": usernames}, Sniper, "") + generator, err := New(map[string]interface{}{"username": usernames, "aliases": moreUsernames}, Sniper, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() count := 0 for { - value, ok := iterator.Value() + _, ok := iterator.Value() if !ok { break } count++ - require.Contains(t, usernames, value["username"], "Could not get correct sniper") } - require.Equal(t, len(usernames), count, "could not get correct sniper counts") + require.Equal(t, len(usernames)+len(moreUsernames), count, "could not get correct sniper counts") } func TestPitchforkGenerator(t *testing.T) { From 07ffe3319aaa43ae6037095daafc9280346a58dd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 14:01:32 +0530 Subject: [PATCH 47/92] Added verbose log support --- v2/pkg/protocols/http/request.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index cd3d562a1..ab30d40df 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -273,6 +273,7 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam e.options.Progress.DecrementRequests(1) return nil, err } + gologger.Verbose().Msgf("Sent request to %s", reqURL) e.options.Output.Request(e.options.TemplateID, reqURL, "http", err) duration := time.Since(timeStart) @@ -318,11 +319,6 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam } } - // var matchData map[string]interface{} - // if payloads != nil { - // matchData = generators.MergeMaps(result.historyData, payloads) - // } - // store for internal purposes the DSL matcher data // hardcode stopping storing data after defaultMaxHistorydata items //if len(result.historyData) < defaultMaxHistorydata { From bbdfb565af33e34bd5ffb7680d885baffec10e5c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 14:54:20 +0530 Subject: [PATCH 48/92] Added network protocol support --- v2/internal/runner/runner.go | 4 + v2/pkg/protocols/network/executer.go | 90 +++++++++++++++ v2/pkg/protocols/network/network.go | 79 +++++++++++++ .../network/networkclientpool/clientpool.go | 38 ++++++ v2/pkg/protocols/network/operators.go | 108 ++++++++++++++++++ v2/pkg/protocols/network/request.go | 96 ++++++++++++++++ v2/pkg/templates/compile.go | 14 ++- v2/pkg/templates/templates.go | 3 + 8 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 v2/pkg/protocols/network/executer.go create mode 100644 v2/pkg/protocols/network/network.go create mode 100644 v2/pkg/protocols/network/networkclientpool/clientpool.go create mode 100644 v2/pkg/protocols/network/operators.go create mode 100644 v2/pkg/protocols/network/request.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f11fde039..9c40f3819 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -277,5 +278,8 @@ func (r *Runner) initializeProtocols() error { if err := httpclientpool.Init(r.options); err != nil { return err } + if err := networkclientpool.Init(r.options); err != nil { + return err + } return nil } diff --git a/v2/pkg/protocols/network/executer.go b/v2/pkg/protocols/network/executer.go new file mode 100644 index 000000000..52f59eb5e --- /dev/null +++ b/v2/pkg/protocols/network/executer.go @@ -0,0 +1,90 @@ +package network + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Executer executes a group of requests for a protocol +type Executer struct { + requests []*Request + options *protocols.ExecuterOptions +} + +var _ protocols.Executer = &Executer{} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { + return &Executer{requests: requests, options: options} +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + for _, request := range e.requests { + err := request.Compile(e.options) + if err != nil { + return err + } + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int { + var count int + for _, request := range e.requests { + count += int(request.Requests()) + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return false, err + } + if events == nil { + return false, nil + } + + // If we have a result field, we should add a result to slice. + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) + } + } + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + var results []*output.InternalWrappedEvent + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return nil, err + } + if events == nil { + return nil, nil + } + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + event.Results = req.makeResultEvent(event) + } + results = append(results, events...) + } + return results, nil +} diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go new file mode 100644 index 000000000..234aed491 --- /dev/null +++ b/v2/pkg/protocols/network/network.go @@ -0,0 +1,79 @@ +package network + +import ( + "net" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" +) + +// Request contains a Network protocol request to be made from a template +type Request struct { + // Address is the address to send requests to (host:port combos generally) + Address string `yaml:"address"` + addressHost string + addressPort string + + // Payload is the payload to send for the network request + Payload string `yaml:"payload"` + // ReadSize is the size of response to read (1024 if not provided by default) + ReadSize int `yaml:"read-size"` + + // Operators for the current request go here. + operators.Operators `yaml:",inline"` + CompiledOperators *operators.Operators + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + var err error + if strings.Contains(r.Address, ":") { + r.addressHost, r.addressPort, err = net.SplitHostPort(r.Address) + if err != nil { + return errors.Wrap(err, "could not parse address") + } + } else { + r.addressHost = r.Address + } + + // Create a client for the class + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + r.options = options + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + return 1 +} + +// Make returns the request to be sent for the protocol +func (r *Request) Make(data string) (string, error) { + replacer := replacer.New(map[string]interface{}{"Address": data}) + address := replacer.Replace(r.addressHost) + if !strings.Contains(address, ":") { + address = net.JoinHostPort(address, r.addressPort) + } + return address, nil +} diff --git a/v2/pkg/protocols/network/networkclientpool/clientpool.go b/v2/pkg/protocols/network/networkclientpool/clientpool.go new file mode 100644 index 000000000..334bcea19 --- /dev/null +++ b/v2/pkg/protocols/network/networkclientpool/clientpool.go @@ -0,0 +1,38 @@ +package networkclientpool + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var ( + normalClient *fastdialer.Dialer +) + +// Init initializes the clientpool implementation +func Init(options *types.Options) error { + // Don't create clients if already created in past. + if normalClient != nil { + return nil + } + dialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions) + if err != nil { + return errors.Wrap(err, "could not create dialer") + } + normalClient = dialer + return nil +} + +// Configuration contains the custom configuration options for a client +type Configuration struct{} + +// Hash returns the hash of the configuration to allow client pooling +func (c *Configuration) Hash() string { + return "" +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (*fastdialer.Dialer, error) { + return normalClient, nil +} diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go new file mode 100644 index 000000000..8d85ee196 --- /dev/null +++ b/v2/pkg/protocols/network/operators.go @@ -0,0 +1,108 @@ +package network + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partString := matcher.Part + switch partString { + case "body", "all", "": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(itemStr))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(itemStr)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(itemStr)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(itemStr)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "body", "all": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent { + data := make(output.InternalEvent, 4) + + // Some data regarding the request metadata + data["host"] = host + data["matched"] = matched + if r.options.Options.JSONRequests { + data["request"] = req + } + data["data"] = resp + return data +} + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "network", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Request = wrapped.InternalEvent["request"].(string) + data.Response = wrapped.InternalEvent["data"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go new file mode 100644 index 000000000..2b84b1a75 --- /dev/null +++ b/v2/pkg/protocols/network/request.go @@ -0,0 +1,96 @@ +package network + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +var _ protocols.Request = &Request{} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { + address, err := getAddress(input) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not get address from url") + } + + // Compile each request for the template based on the URL + actualAddress, err := r.Make(address) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not build request") + } + + conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not connect to server request") + } + defer conn.Close() + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + _, err = conn.Write([]byte(r.Payload)) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not write request to server") + } + r.options.Progress.IncrementRequests() + + r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress) + + if r.options.Options.Debug { + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + fmt.Fprintf(os.Stderr, "%s\n", r.Payload) + } + + bufferSize := 1024 + if r.ReadSize != 0 { + bufferSize = r.ReadSize + } + buffer := make([]byte, bufferSize) + n, _ := conn.Read(buffer) + resp := string(buffer[:n]) + + if r.options.Options.Debug { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) + fmt.Fprintf(os.Stderr, "%s\n", resp) + } + ouputEvent := r.responseToDSLMap(r.Payload, resp, input, actualAddress) + + event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + if r.CompiledOperators != nil { + result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + if !ok { + return nil, nil + } + event[0].OperatorsResult = result + } + return event, nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if strings.Contains(toTest, "://") { + parsed, err := url.Parse(toTest) + if err != nil { + return "", err + } + toTest = parsed.Host + } + return toTest, nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 8f18df1dc..e3f3e1468 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -32,12 +33,8 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { options.TemplateInfo = template.Info options.TemplatePath = file - // We don't support both http and dns in a single template - if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 { - return nil, fmt.Errorf("both http and dns requests for %s", template.ID) - } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsNetwork)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -58,6 +55,9 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { for _, request := range template.RequestsHTTP { template.TotalRequests += request.Requests() } + for _, request := range template.RequestsNetwork { + template.TotalRequests += request.Requests() + } if len(template.RequestsDNS) > 0 { template.Executer = dns.NewExecuter(template.RequestsDNS, options) err = template.Executer.Compile() @@ -66,6 +66,10 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { template.Executer = http.NewExecuter(template.RequestsHTTP, options) err = template.Executer.Compile() } + if len(template.RequestsNetwork) > 0 { + template.Executer = network.NewExecuter(template.RequestsNetwork, options) + err = template.Executer.Compile() + } if err != nil { return nil, errors.Wrap(err, "could not compile request") } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 8b54d9f0b..f4c341eaf 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -4,6 +4,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -17,6 +18,8 @@ type Template struct { RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` + // RequestsNetwork contains the network request to make in the template + RequestsNetwork []*network.Request `yaml:"network,omitempty"` // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline"` CompiledWorkflow *workflows.Workflow From 55e9a9acfe90b6f4e75cdd364efaefff60c7c4b4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 16:47:22 +0530 Subject: [PATCH 49/92] Added new payload type with hex support --- v2/pkg/protocols/network/network.go | 10 +++++++- v2/pkg/protocols/network/request.go | 39 +++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 234aed491..4e601423f 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -20,7 +20,7 @@ type Request struct { addressPort string // Payload is the payload to send for the network request - Payload string `yaml:"payload"` + Inputs []*Input `yaml:"inputs"` // ReadSize is the size of response to read (1024 if not provided by default) ReadSize int `yaml:"read-size"` @@ -33,6 +33,14 @@ type Request struct { options *protocols.ExecuterOptions } +// Input is the input to send on the network +type Input struct { + // Data is the data to send as the input + Data string `yaml:"data"` + // Type is the type of input - hex, text. + Type string `yaml:"type"` +} + // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { var err error diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 2b84b1a75..e01cf19ec 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -2,6 +2,7 @@ package network import ( "context" + "encoding/hex" "fmt" "net/url" "os" @@ -42,22 +43,46 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent defer conn.Close() conn.SetReadDeadline(time.Now().Add(5 * time.Second)) - _, err = conn.Write([]byte(r.Payload)) + reqBuilder := &strings.Builder{} + for _, input := range r.Inputs { + var data []byte + + switch input.Type { + case "hex": + data, err = hex.DecodeString(input.Data) + default: + data = []byte(input.Data) + } + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not write request to server") + } + reqBuilder.Grow(len(input.Data)) + reqBuilder.WriteString(input.Data) + + _, err = conn.Write(data) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not write request to server") + } + r.options.Progress.IncrementRequests() + } if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) return nil, errors.Wrap(err, "could not write request to server") } - r.options.Progress.IncrementRequests() - - r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) - gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress) if r.options.Options.Debug { gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) - fmt.Fprintf(os.Stderr, "%s\n", r.Payload) + fmt.Fprintf(os.Stderr, "%s\n", reqBuilder.String()) } + r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress) + bufferSize := 1024 if r.ReadSize != 0 { bufferSize = r.ReadSize @@ -70,7 +95,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) fmt.Fprintf(os.Stderr, "%s\n", resp) } - ouputEvent := r.responseToDSLMap(r.Payload, resp, input, actualAddress) + ouputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} if r.CompiledOperators != nil { From 6caffb4575da5cfd2c7d6b57aabb729aadf002a2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 16:49:45 +0530 Subject: [PATCH 50/92] Fixed panic in parallel requests --- v2/pkg/protocols/http/request.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index ab30d40df..defd26731 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -33,7 +33,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int swg := sizedwaitgroup.New(maxWorkers) var requestErr error - var mutex *sync.Mutex + mutex := &sync.Mutex{} var outputs []*output.InternalWrappedEvent for i := 0; i < e.RaceNumberRequests; i++ { request, err := generator.Make(reqURL, nil) @@ -67,7 +67,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in swg := sizedwaitgroup.New(maxWorkers) var requestErr error - var mutex *sync.Mutex + mutex := &sync.Mutex{} var outputs []*output.InternalWrappedEvent for { request, err := generator.Make(reqURL, dynamicValues) @@ -128,7 +128,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter swg := sizedwaitgroup.New(maxWorkers) var requestErr error - var mutex *sync.Mutex + mutex := &sync.Mutex{} var outputs []*output.InternalWrappedEvent for { request, err := generator.Make(reqURL, dynamicValues) From 99d16a4d024b303cfbdca0bc2baf383fa8e30223 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 21:14:04 +0530 Subject: [PATCH 51/92] More work on network protocol support + misc fixes --- v2/internal/runner/config.go | 22 --------- v2/internal/runner/processor.go | 1 + v2/pkg/output/output.go | 1 - v2/pkg/protocols/http/raw/raw.go | 1 - v2/pkg/protocols/http/request.go | 75 ++++++++++++++--------------- v2/pkg/protocols/network/network.go | 39 +++++++-------- v2/pkg/protocols/network/request.go | 33 +++++++++++-- v2/pkg/templates/compile.go | 16 +----- v2/pkg/workflows/execute_test.go | 22 ++++----- 9 files changed, 98 insertions(+), 112 deletions(-) diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index db739dd50..4c786dcf5 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -93,28 +93,6 @@ func (r *Runner) readNucleiIgnoreFile() { } } -// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. -func (r *Runner) checkIfInNucleiIgnore(item string) bool { - if r.templatesConfig == nil { - return false - } - - for _, paths := range r.templatesConfig.IgnorePaths { - // If we have a path to ignore, check if it's in the item. - if paths[len(paths)-1] == '/' { - if strings.Contains(item, paths) { - return true - } - continue - } - // Check for file based extension in ignores - if strings.HasSuffix(item, paths) { - return true - } - } - return false -} - // getIgnoreFilePath returns the ignore file path for the runner func (r *Runner) getIgnoreFilePath() string { defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index fc9d6c9b2..4ed008cd5 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -18,6 +18,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) + wg.Add() go func(URL string) { defer wg.Done() diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 97b546967..2faab9a62 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -161,7 +161,6 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error) return } w.traceMutex.Lock() - //nolint:errcheck // We don't need to do anything here _ = w.traceFile.Write(data) w.traceMutex.Unlock() } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index b2d5eeab1..c14bf3999 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -47,7 +47,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { break } - //nolint:gomnd // this is not a magic number p := strings.SplitN(line, ":", 2) key = p[0] if len(p) > 1 { diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index ab30d40df..4d68ce35f 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -161,23 +161,23 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter } // ExecuteWithResults executes the final request on a URL -func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // verify if pipeline was requested - if e.Pipeline { - return e.executeTurboHTTP(reqURL, dynamicValues) + if r.Pipeline { + return r.executeTurboHTTP(reqURL, dynamicValues) } // verify if a basic race condition was requested - if e.Race && e.RaceNumberRequests > 0 { - return e.executeRaceRequest(reqURL, dynamicValues) + if r.Race && r.RaceNumberRequests > 0 { + return r.executeRaceRequest(reqURL, dynamicValues) } // verify if parallel elaboration was requested - if e.Threads > 0 { - return e.executeParallelHTTP(reqURL, dynamicValues) + if r.Threads > 0 { + return r.executeParallelHTTP(reqURL, dynamicValues) } - generator := e.newGenerator() + generator := r.newGenerator() var requestErr error var outputs []*output.InternalWrappedEvent @@ -187,21 +187,21 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int break } if err != nil { - e.options.Progress.DecrementRequests(int64(generator.Total())) + r.options.Progress.DecrementRequests(int64(generator.Total())) return nil, err } - e.options.RateLimiter.Take() - output, err := e.executeRequest(reqURL, request, dynamicValues) + r.options.RateLimiter.Take() + output, err := r.executeRequest(reqURL, request, dynamicValues) if err != nil { requestErr = multierr.Append(requestErr, err) } else { outputs = append(outputs, output...) } - e.options.Progress.IncrementRequests() + r.options.Progress.IncrementRequests() if request.original.options.Options.StopAtFirstMatch && len(output) > 0 { - e.options.Progress.DecrementRequests(int64(generator.Total())) + r.options.Progress.DecrementRequests(int64(generator.Total())) break } } @@ -209,16 +209,15 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int } // executeRequest executes the actual generated request and returns error if occured -func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given - if e.options.Options.RandomAgent { + if r.options.Options.RandomAgent { builder := &strings.Builder{} builder.WriteString("User-Agent: ") - // nolint:errcheck // ignoring error builder.WriteString(uarand.GetRandom()) - e.customHeaders.Set(builder.String()) + r.customHeaders.Set(builder.String()) } - e.setCustomHeaders(request) + r.setCustomHeaders(request) var ( resp *http.Response @@ -226,14 +225,14 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam dumpedRequest []byte fromcache bool ) - if e.options.Options.Debug || e.options.ProjectFile != nil { + if r.options.Options.Debug || r.options.ProjectFile != nil { dumpedRequest, err = dump(request, reqURL) if err != nil { return nil, err } } - if e.options.Options.Debug { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", e.options.TemplateID, reqURL) + if r.options.Options.Debug { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } @@ -245,23 +244,23 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam // burp uses "\r\n" as new line character request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n") options := request.original.rawhttpClient.Options - options.AutomaticContentLength = !e.DisableAutoContentLength - options.AutomaticHostHeader = !e.DisableAutoHostname - options.FollowRedirects = e.Redirects + options.AutomaticContentLength = !r.DisableAutoContentLength + options.AutomaticHostHeader = !r.DisableAutoHostname + options.FollowRedirects = r.Redirects resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) } else { // if nuclei-project is available check if the request was already sent previously - if e.options.ProjectFile != nil { + if r.options.ProjectFile != nil { // if unavailable fail silently fromcache = true // nolint:bodyclose // false positive the response is generated at runtime - resp, err = e.options.ProjectFile.Get(dumpedRequest) + resp, err = r.options.ProjectFile.Get(dumpedRequest) if err != nil { fromcache = false } } if resp == nil { - resp, err = e.httpClient.Do(request.request) + resp, err = r.httpClient.Do(request.request) } } if err != nil { @@ -269,17 +268,17 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } - e.options.Output.Request(e.options.TemplateID, reqURL, "http", err) - e.options.Progress.DecrementRequests(1) + r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) + r.options.Progress.DecrementRequests(1) return nil, err } gologger.Verbose().Msgf("Sent request to %s", reqURL) - e.options.Output.Request(e.options.TemplateID, reqURL, "http", err) + r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) duration := time.Since(timeStart) // Dump response - Step 1 - Decompression not yet handled var dumpedResponse []byte - if e.options.Options.Debug { + if r.options.Options.Debug { var dumpErr error dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { @@ -305,15 +304,15 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam } // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if e.options.Options.Debug { + if r.options.Options.Debug { dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", e.options.TemplateID, reqURL) + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, reqURL) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) } // if nuclei-project is enabled store the response if not previously done - if e.options.ProjectFile != nil && !fromcache { - err := e.options.ProjectFile.Set(dumpedRequest, resp, data) + if r.options.ProjectFile != nil && !fromcache { + err := r.options.ProjectFile.Set(dumpedRequest, resp, data) if err != nil { return nil, errors.Wrap(err, "could not store in project file") } @@ -346,11 +345,11 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam if request.request != nil { matchedURL = request.request.URL.String() } - ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) + ouputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} - if e.CompiledOperators != nil { - result, ok := e.Operators.Execute(ouputEvent, e.Match, e.Extract) + if r.CompiledOperators != nil { + result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { return nil, nil } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 4e601423f..f68f22822 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -8,16 +8,14 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" ) // Request contains a Network protocol request to be made from a template type Request struct { // Address is the address to send requests to (host:port combos generally) - Address string `yaml:"address"` - addressHost string - addressPort string + Address []string `yaml:"host"` + addresses []keyValue // Payload is the payload to send for the network request Inputs []*Input `yaml:"inputs"` @@ -33,6 +31,12 @@ type Request struct { options *protocols.ExecuterOptions } +// keyValue is a key value pair +type keyValue struct { + key string + value string +} + // Input is the input to send on the network type Input struct { // Data is the data to send as the input @@ -44,13 +48,16 @@ type Input struct { // Compile compiles the protocol request for further execution. func (r *Request) Compile(options *protocols.ExecuterOptions) error { var err error - if strings.Contains(r.Address, ":") { - r.addressHost, r.addressPort, err = net.SplitHostPort(r.Address) - if err != nil { - return errors.Wrap(err, "could not parse address") + for _, address := range r.Address { + if strings.Contains(address, ":") { + addressHost, addressPort, err := net.SplitHostPort(address) + if err != nil { + return errors.Wrap(err, "could not parse address") + } + r.addresses = append(r.addresses, keyValue{key: addressHost, value: addressPort}) + } else { + r.addresses = append(r.addresses, keyValue{key: address}) } - } else { - r.addressHost = r.Address } // Create a client for the class @@ -73,15 +80,5 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Requests returns the total number of requests the YAML rule will perform func (r *Request) Requests() int { - return 1 -} - -// Make returns the request to be sent for the protocol -func (r *Request) Make(data string) (string, error) { - replacer := replacer.New(map[string]interface{}{"Address": data}) - address := replacer.Replace(r.addressHost) - if !strings.Contains(address, ":") { - address = net.JoinHostPort(address, r.addressPort) - } - return address, nil + return len(r.addresses) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index e01cf19ec..8bbbc01a0 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "fmt" + "net" "net/url" "os" "strings" @@ -13,6 +14,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) var _ protocols.Request = &Request{} @@ -26,12 +28,34 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent return nil, errors.Wrap(err, "could not get address from url") } - // Compile each request for the template based on the URL - actualAddress, err := r.Make(address) - if err != nil { + var outputs []*output.InternalWrappedEvent + for _, kv := range r.addresses { + replacer := replacer.New(map[string]interface{}{"Hostname": address}) + actualAddress := replacer.Replace(kv.key) + if kv.value != "" { + if strings.Contains(address, ":") { + actualAddress, _, _ = net.SplitHostPort(actualAddress) + } + actualAddress = net.JoinHostPort(actualAddress, kv.value) + } + + output, err := r.executeAddress(actualAddress, address, input) + if err != nil { + gologger.Error().Msgf("Could not make network request for %s: %s\n", actualAddress, err) + continue + } + outputs = append(outputs, output...) + } + return outputs, nil +} + +// executeAddress executes the request for an address +func (r *Request) executeAddress(actualAddress, address, input string) ([]*output.InternalWrappedEvent, error) { + if !strings.Contains(actualAddress, ":") { + err := errors.New("no port provided in network protocol request") r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not build request") + return nil, err } conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress) @@ -77,6 +101,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent if r.options.Options.Debug { gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + fmt.Fprintf(os.Stderr, "%s\n", reqBuilder.String()) } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index e3f3e1468..ae537d235 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -49,15 +49,6 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { } // Compile the requests found - for _, request := range template.RequestsDNS { - template.TotalRequests += request.Requests() - } - for _, request := range template.RequestsHTTP { - template.TotalRequests += request.Requests() - } - for _, request := range template.RequestsNetwork { - template.TotalRequests += request.Requests() - } if len(template.RequestsDNS) > 0 { template.Executer = dns.NewExecuter(template.RequestsDNS, options) err = template.Executer.Compile() @@ -70,14 +61,11 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { template.Executer = network.NewExecuter(template.RequestsNetwork, options) err = template.Executer.Compile() } + template.TotalRequests += template.Executer.Requests() + if err != nil { return nil, errors.Wrap(err, "could not compile request") } - if template.Executer != nil { - if err := template.Executer.Compile(); err != nil { - return nil, errors.Wrap(err, "could not compile template executer") - } - } return template, nil } diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 47120be2e..dfe1f4036 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -10,7 +10,7 @@ import ( func TestWorkflowsSimple(t *testing.T) { workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true}}, + {Executer: &mockExecuter{result: true}}, }} matched, err := workflow.RunWorkflow("https://test.com") @@ -21,10 +21,10 @@ func TestWorkflowsSimple(t *testing.T) { func TestWorkflowsSimpleMultiple(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }}}, - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, }} @@ -40,11 +40,11 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { func TestWorkflowsSubtemplates(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }}, Subtemplates: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, }}, @@ -61,11 +61,11 @@ func TestWorkflowsSubtemplates(t *testing.T) { func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: false, executeHook: func(input string) { + {Executer: &mockExecuter{result: false, executeHook: func(input string) { firstInput = input }}, Subtemplates: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, }}, @@ -82,7 +82,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ @@ -92,7 +92,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }}, Matchers: []*Matcher{ {Name: "tomcat", Subtemplates: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, }}, @@ -111,7 +111,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ @@ -121,7 +121,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }}, Matchers: []*Matcher{ {Name: "apache", Subtemplates: []*WorkflowTemplate{ - {executer: &mockExecuter{result: true, executeHook: func(input string) { + {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, }}, From 590f8042b9b5f00c07a2d8067a89a5270393f965 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 23:20:31 +0530 Subject: [PATCH 52/92] Don't log without verbose --- v2/pkg/protocols/network/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 8bbbc01a0..f33e43248 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -41,7 +41,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent output, err := r.executeAddress(actualAddress, address, input) if err != nil { - gologger.Error().Msgf("Could not make network request for %s: %s\n", actualAddress, err) + gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } outputs = append(outputs, output...) From 33e1d4ddb91d7c6c63685a41d297239d2d8bf2b8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 15:28:28 +0530 Subject: [PATCH 53/92] Added file based template support --- v2/pkg/protocols/common/tostring/tostring.go | 8 ++ v2/pkg/protocols/file/executer.go | 90 ++++++++++++++ v2/pkg/protocols/file/file.go | 73 ++++++++++++ v2/pkg/protocols/file/find.go | 117 +++++++++++++++++++ v2/pkg/protocols/file/operators.go | 104 +++++++++++++++++ v2/pkg/protocols/file/request.go | 70 +++++++++++ v2/pkg/protocols/http/request.go | 3 +- v2/pkg/protocols/http/utils.go | 6 - v2/pkg/templates/compile.go | 21 ++-- v2/pkg/templates/templates.go | 3 + 10 files changed, 478 insertions(+), 17 deletions(-) create mode 100644 v2/pkg/protocols/common/tostring/tostring.go create mode 100644 v2/pkg/protocols/file/executer.go create mode 100644 v2/pkg/protocols/file/file.go create mode 100644 v2/pkg/protocols/file/find.go create mode 100644 v2/pkg/protocols/file/operators.go create mode 100644 v2/pkg/protocols/file/request.go diff --git a/v2/pkg/protocols/common/tostring/tostring.go b/v2/pkg/protocols/common/tostring/tostring.go new file mode 100644 index 000000000..d22d2bf56 --- /dev/null +++ b/v2/pkg/protocols/common/tostring/tostring.go @@ -0,0 +1,8 @@ +package tostring + +import "unsafe" + +// UnsafeToString converts byte slice to string with zero allocations +func UnsafeToString(bs []byte) string { + return *(*string)(unsafe.Pointer(&bs)) +} diff --git a/v2/pkg/protocols/file/executer.go b/v2/pkg/protocols/file/executer.go new file mode 100644 index 000000000..29e508173 --- /dev/null +++ b/v2/pkg/protocols/file/executer.go @@ -0,0 +1,90 @@ +package file + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Executer executes a group of requests for a protocol +type Executer struct { + requests []*Request + options *protocols.ExecuterOptions +} + +var _ protocols.Executer = &Executer{} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { + return &Executer{requests: requests, options: options} +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + for _, request := range e.requests { + err := request.Compile(e.options) + if err != nil { + return err + } + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int { + var count int + for _, request := range e.requests { + count += request.Requests() + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return false, err + } + if events == nil { + return false, nil + } + + // If we have a result field, we should add a result to slice. + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) + } + } + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + var results []*output.InternalWrappedEvent + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return nil, err + } + if events == nil { + return nil, nil + } + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + event.Results = req.makeResultEvent(event) + } + results = append(results, events...) + } + return results, nil +} diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go new file mode 100644 index 000000000..62b19290b --- /dev/null +++ b/v2/pkg/protocols/file/file.go @@ -0,0 +1,73 @@ +package file + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Request contains a File matching mechanism for local disk operations. +type Request struct { + // MaxSize is the maximum size of the file to run request on. + // By default, nuclei will process 5MB files and not go more than that. + // It can be set to much lower or higher depending on use. + MaxSize int `yaml:"max-size"` + // NoRecursive specifies whether to not do recursive checks if folders are provided. + NoRecursive bool `yaml:"no-recursive"` + // Extensions is the list of extensions to perform matching on. + Extensions []string `yaml:"extensions"` + // ExtensionAllowlist is the list of file extensions to enforce allowing. + ExtensionAllowlist []string `yaml:"allowlist"` + // ExtensionDenylist is the list of file extensions to deny during matching. + ExtensionDenylist []string `yaml:"denylist"` + + // Operators for the current request go here. + operators.Operators `yaml:",inline"` + CompiledOperators *operators.Operators + + // cache any variables that may be needed for operation. + options *protocols.ExecuterOptions + extensions map[string]struct{} + extensionDenylist map[string]struct{} +} + +// defaultDenylist is the default list of extensions to be denied +var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + // By default use 5mb as max size to read. + if r.MaxSize == 0 { + r.MaxSize = 5 * 1024 * 1024 + } + r.options = options + + r.extensions = make(map[string]struct{}) + r.extensionDenylist = make(map[string]struct{}) + + for _, extension := range r.Extensions { + r.extensions[extension] = struct{}{} + } + for _, extension := range defaultDenylist { + r.extensionDenylist[extension] = struct{}{} + } + for _, extension := range r.ExtensionDenylist { + r.extensionDenylist[extension] = struct{}{} + } + for _, extension := range r.ExtensionAllowlist { + delete(r.extensionDenylist, extension) + } + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + return 1 +} diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go new file mode 100644 index 000000000..2273cb769 --- /dev/null +++ b/v2/pkg/protocols/file/find.go @@ -0,0 +1,117 @@ +package file + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" +) + +// getInputPaths parses the specified input paths and returns a compiled +// list of finished absolute paths to the files evaluating any allowlist, denylist, +// glob, file or folders, etc. +func (r *Request) getInputPaths(target string, callback func(string)) error { + processed := make(map[string]struct{}) + + // Template input includes a wildcard + if strings.Contains(target, "*") { + err := r.findGlobPathMatches(target, processed, callback) + if err != nil { + return errors.Wrap(err, "could not find glob matches") + } + return nil + } + + // Template input is either a file or a directory + file, err := r.findFileMatches(target, processed, callback) + if err != nil { + return errors.Wrap(err, "could not find file") + } + if file { + return nil + } + + // Recursively walk down the Templates directory and run all + // the template file checks + err = r.findDirectoryMatches(target, processed, callback) + if err != nil { + return errors.Wrap(err, "could not find directory matches") + } + return nil +} + +// findGlobPathMatches returns the matched files from a glob path +func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { + matches, err := filepath.Glob(absPath) + if err != nil { + return errors.Errorf("wildcard found, but unable to glob: %s\n", err) + } + for _, match := range matches { + if !r.validatePath(match) { + continue + } + if _, ok := processed[match]; !ok { + processed[match] = struct{}{} + callback(match) + } + } + return nil +} + +// findFileMatches finds if a path is an absolute file. If the path +// is a file, it returns true otherwise false with no errors. +func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { + info, err := os.Stat(absPath) + if err != nil { + return false, err + } + if !info.Mode().IsRegular() { + return false, nil + } + if _, ok := processed[absPath]; !ok { + processed[absPath] = struct{}{} + callback(absPath) + } + return true, nil +} + +// findDirectoryMatches finds matches for templates from a directory +func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { + err := godirwalk.Walk(absPath, &godirwalk.Options{ + Unsorted: true, + ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { + return godirwalk.SkipNode + }, + Callback: func(path string, d *godirwalk.Dirent) error { + if !r.validatePath(path) { + return nil + } + if _, ok := processed[path]; !ok { + callback(path) + processed[path] = struct{}{} + } + return nil + }, + }) + return err +} + +// validatePath validates a file path for blacklist and whitelist options +func (r *Request) validatePath(item string) bool { + extension := path.Ext(item) + if len(r.extensions) > 0 { + if _, ok := r.extensions[extension]; ok { + return true + } + return false + } + if _, ok := r.extensionDenylist[extension]; ok { + gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension) + return false + } + return true +} diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go new file mode 100644 index 000000000..894f833e4 --- /dev/null +++ b/v2/pkg/protocols/file/operators.go @@ -0,0 +1,104 @@ +package file + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partString := matcher.Part + switch partString { + case "body", "all", "": + partString = "raw" + } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(itemStr))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(itemStr)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(itemStr)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(itemStr)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "body", "all", "": + partString = "raw" + } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func (r *Request) responseToDSLMap(raw string, host, matched string) output.InternalEvent { + data := make(output.InternalEvent, 3) + + // Some data regarding the request metadata + data["host"] = host + data["matched"] = matched + data["raw"] = raw + return data +} + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "file", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Response = wrapped.InternalEvent["raw"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go new file mode 100644 index 000000000..f5b21bddd --- /dev/null +++ b/v2/pkg/protocols/file/request.go @@ -0,0 +1,70 @@ +package file + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" +) + +var _ protocols.Request = &Request{} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { + var events []*output.InternalWrappedEvent + + err := r.getInputPaths(input, func(data string) { + file, err := os.Open(data) + if err != nil { + gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + return + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + return + } + if stat.Size() >= int64(r.MaxSize) { + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + return + } + + buffer, err := ioutil.ReadAll(file) + if err != nil { + gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + return + } + dataStr := tostring.UnsafeToString(buffer) + + if r.options.Options.Debug { + gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) + fmt.Fprintf(os.Stderr, "%s\n", dataStr) + } + gologger.Verbose().Msgf("[%s] Sent file request to %s", r.options.TemplateID, data) + ouputEvent := r.responseToDSLMap(dataStr, input, data) + + event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + if r.CompiledOperators != nil { + result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + if !ok { + return + } + event[0].OperatorsResult = result + } + events = append(events, event...) + }) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "file", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not send file request") + } + r.options.Progress.IncrementRequests() + return events, nil +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index defd26731..aa9f00f7f 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" @@ -346,7 +347,7 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam if request.request != nil { matchedURL = request.request.URL.String() } - ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta) + ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} if e.CompiledOperators != nil { diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index 5629307c9..f0eb4ace0 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -7,17 +7,11 @@ import ( "net/http" "net/http/httputil" "strings" - "unsafe" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/rawhttp" ) -// unsafeToString converts byte slice to string with zero allocations -func unsafeToString(bs []byte) string { - return *(*string)(unsafe.Pointer(&bs)) -} - // headersToString converts http headers to string func headersToString(headers http.Header) string { builder := &strings.Builder{} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 8f18df1dc..bdd2cce2f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -7,16 +7,17 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) // Parse parses a yaml request template file -func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { +func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, error) { template := &Template{} - f, err := os.Open(file) + f, err := os.Open(filePath) if err != nil { return nil, err } @@ -30,14 +31,10 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { // Setting up variables regarding template metadata options.TemplateID = template.ID options.TemplateInfo = template.Info - options.TemplatePath = file + options.TemplatePath = filePath - // We don't support both http and dns in a single template - if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 { - return nil, fmt.Errorf("both http and dns requests for %s", template.ID) - } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -58,13 +55,17 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { for _, request := range template.RequestsHTTP { template.TotalRequests += request.Requests() } + for _, request := range template.RequestsFile { + template.TotalRequests += request.Requests() + } if len(template.RequestsDNS) > 0 { template.Executer = dns.NewExecuter(template.RequestsDNS, options) - err = template.Executer.Compile() } if len(template.RequestsHTTP) > 0 { template.Executer = http.NewExecuter(template.RequestsHTTP, options) - err = template.Executer.Compile() + } + if len(template.RequestsFile) > 0 { + template.Executer = file.NewExecuter(template.RequestsFile, options) } if err != nil { return nil, errors.Wrap(err, "could not compile request") diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 8b54d9f0b..bf09cd1c3 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -3,6 +3,7 @@ package templates import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -17,6 +18,8 @@ type Template struct { RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` + // RequestsFile contains the file request to make in the template + RequestsFile []*file.Request `yaml:"file,omitempty"` // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline"` CompiledWorkflow *workflows.Workflow From d106ae2ef1bbfb05ddac92cc46a73950476de846 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 15:31:44 +0530 Subject: [PATCH 54/92] Small change to extensions --- v2/pkg/protocols/file/file.go | 7 ++++++- v2/pkg/protocols/file/find.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 62b19290b..0056a271f 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -28,6 +28,7 @@ type Request struct { // cache any variables that may be needed for operation. options *protocols.ExecuterOptions extensions map[string]struct{} + allExtensions bool extensionDenylist map[string]struct{} } @@ -53,7 +54,11 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.extensionDenylist = make(map[string]struct{}) for _, extension := range r.Extensions { - r.extensions[extension] = struct{}{} + if extension == "*" { + r.allExtensions = true + } else { + r.extensions[extension] = struct{}{} + } } for _, extension := range defaultDenylist { r.extensionDenylist[extension] = struct{}{} diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 2273cb769..210fb6d47 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -103,7 +103,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru // validatePath validates a file path for blacklist and whitelist options func (r *Request) validatePath(item string) bool { extension := path.Ext(item) - if len(r.extensions) > 0 { + if len(r.extensions) > 0 && !r.allExtensions { if _, ok := r.extensions[extension]; ok { return true } From bf63eb5937705cc93c012d9bf0e6dcafd378deae Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 16:52:41 +0530 Subject: [PATCH 55/92] Add dynamic extracted values to history --- v2/pkg/operators/operators.go | 4 ++-- v2/pkg/protocols/http/request.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 12fce8d1f..bdbe82332 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -56,7 +56,7 @@ type Result struct { // OutputExtracts is the list of extracts to be displayed on screen. OutputExtracts []string // DynamicValues contains any dynamic values to be templated - DynamicValues map[string]string + DynamicValues map[string]interface{} // PayloadValues contains payload values provided by user. (Optional) PayloadValues map[string]interface{} } @@ -75,7 +75,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac result := &Result{ Matches: make(map[string]struct{}), Extracts: make(map[string][]string), - DynamicValues: make(map[string]string), + DynamicValues: make(map[string]interface{}), } for _, matcher := range r.Matchers { // Check if the matcher matched diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index aa9f00f7f..3af6519c1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -197,6 +197,12 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int if err != nil { requestErr = multierr.Append(requestErr, err) } else { + // Add the extracts to the dynamic values if any. + for _, o := range output { + if o.OperatorsResult != nil { + dynamicValues = generators.MergeMaps(dynamicValues, o.OperatorsResult.DynamicValues) + } + } outputs = append(outputs, output...) } e.options.Progress.IncrementRequests() From 65e14e1c9176b6b3c06ba22ea289b631bfdde300 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 17:07:24 +0530 Subject: [PATCH 56/92] Fixed not working part:all in http requests --- v2/pkg/protocols/http/operators.go | 39 ++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a3e15d898..5eaf239f9 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -18,15 +18,22 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) switch partString { case "header": partString = "all_headers" - case "all": - partString = "raw" } - item, ok := data[partString] - if !ok { - return false + var itemStr string + if partString == "all" { + builder := &strings.Builder{} + builder.WriteString(data["body"].(string)) + builder.WriteString("\n\n") + builder.WriteString(data["all_headers"].(string)) + itemStr = builder.String() + } else { + item, ok := data[partString] + if !ok { + return false + } + itemStr = types.ToString(item) } - itemStr := types.ToString(item) switch matcher.GetType() { case matchers.StatusMatcher: @@ -55,16 +62,22 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext switch partString { case "header": partString = "all_headers" - case "all": - partString = "raw" } - item, ok := data[partString] - if !ok { - return nil + var itemStr string + if partString == "all" { + builder := &strings.Builder{} + builder.WriteString(data["body"].(string)) + builder.WriteString("\n\n") + builder.WriteString(data["all_headers"].(string)) + itemStr = builder.String() + } else { + item, ok := data[partString] + if !ok { + return nil + } + itemStr = types.ToString(item) } - itemStr := types.ToString(item) - switch extractor.GetType() { case extractors.RegexExtractor: return extractor.ExtractRegex(itemStr) From 3dc82c95d4c7e28cdca39749fed382201467c3ea Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 19:05:40 +0530 Subject: [PATCH 57/92] Fixed bug with internal extracts being marked as results --- v2/pkg/operators/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index bdbe82332..9e9f13375 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -110,7 +110,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac result.OutputExtracts = append(result.OutputExtracts, match) } } - if len(extractorResults) > 0 { + if len(extractorResults) > 0 && !extractor.Internal { result.Extracts[extractor.Name] = extractorResults } } From 370ded871cfd57293d2350b362dffd674dc267b6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 1 Jan 2021 19:36:21 +0530 Subject: [PATCH 58/92] Added support for output streaming in nuclei --- v2/pkg/protocols/dns/executer.go | 37 +++-------- v2/pkg/protocols/dns/request.go | 15 ++--- v2/pkg/protocols/file/executer.go | 37 +++-------- v2/pkg/protocols/file/request.go | 14 ++--- v2/pkg/protocols/http/executer.go | 40 ++++-------- v2/pkg/protocols/http/operators.go | 76 ++++++++++------------- v2/pkg/protocols/http/request.go | 92 +++++++++++++--------------- v2/pkg/protocols/network/executer.go | 37 +++-------- v2/pkg/protocols/network/request.go | 31 +++++----- v2/pkg/protocols/protocols.go | 7 ++- v2/pkg/templates/compile.go | 7 ++- v2/pkg/workflows/execute.go | 37 ++++++----- v2/pkg/workflows/execute_test.go | 8 ++- 13 files changed, 180 insertions(+), 258 deletions(-) diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go index 7863efa0a..62216dc25 100644 --- a/v2/pkg/protocols/dns/executer.go +++ b/v2/pkg/protocols/dns/executer.go @@ -43,48 +43,29 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return false, err - } - if events == nil { - return false, nil - } - - // If we have a result field, we should add a result to slice. - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } - for _, result := range req.makeResultEvent(event) { results = true e.options.Output.Write(result) } - } + }) } return results, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { - var results []*output.InternalWrappedEvent - +func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return nil, err - } - if events == nil { - return nil, nil - } - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } event.Results = req.makeResultEvent(event) - } - results = append(results, events...) + callback(event) + }) } - return results, nil + return nil } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 47779cf9f..90785f34f 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -14,7 +14,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -28,7 +28,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent if err != nil { r.options.Output.Request(r.options.TemplateID, domain, "dns", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not build request") + return errors.Wrap(err, "could not build request") } if r.options.Options.Debug { @@ -41,7 +41,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent if err != nil { r.options.Output.Request(r.options.TemplateID, domain, "dns", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not send dns request") + return errors.Wrap(err, "could not send dns request") } r.options.Progress.IncrementRequests() @@ -54,15 +54,16 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent } ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) - event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { - return nil, nil + return nil } - event[0].OperatorsResult = result + event.OperatorsResult = result } - return event, nil + callback(event) + return nil } // isURL tests a string to determine if it is a well-structured url or not. diff --git a/v2/pkg/protocols/file/executer.go b/v2/pkg/protocols/file/executer.go index 29e508173..286f339de 100644 --- a/v2/pkg/protocols/file/executer.go +++ b/v2/pkg/protocols/file/executer.go @@ -43,48 +43,29 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return false, err - } - if events == nil { - return false, nil - } - - // If we have a result field, we should add a result to slice. - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } - for _, result := range req.makeResultEvent(event) { results = true e.options.Output.Write(result) } - } + }) } return results, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { - var results []*output.InternalWrappedEvent - +func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return nil, err - } - if events == nil { - return nil, nil - } - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } event.Results = req.makeResultEvent(event) - } - results = append(results, events...) + callback(event) + }) } - return results, nil + return nil } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index f5b21bddd..cbcf63ef4 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -15,9 +15,7 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { - var events []*output.InternalWrappedEvent - +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { err := r.getInputPaths(input, func(data string) { file, err := os.Open(data) if err != nil { @@ -50,21 +48,21 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent gologger.Verbose().Msgf("[%s] Sent file request to %s", r.options.TemplateID, data) ouputEvent := r.responseToDSLMap(dataStr, input, data) - event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { return } - event[0].OperatorsResult = result + event.OperatorsResult = result + callback(event) } - events = append(events, event...) }) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "file", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not send file request") + return errors.Wrap(err, "could not send file request") } r.options.Progress.IncrementRequests() - return events, nil + return nil } diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/http/executer.go index df92c1c34..d12dad14e 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/http/executer.go @@ -42,49 +42,35 @@ func (e *Executer) Requests() int { func (e *Executer) Execute(input string) (bool, error) { var results bool + dynamicValues := make(map[string]interface{}) for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return false, err - } - if events == nil { - return false, nil - } - - // If we have a result field, we should add a result to slice. - for _, event := range events { + err := req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } - for _, result := range req.makeResultEvent(event) { results = true e.options.Output.Write(result) } + }) + if err != nil { + continue } } return results, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { - var results []*output.InternalWrappedEvent - +func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { + dynamicValues := make(map[string]interface{}) for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return nil, err - } - if events == nil { - return nil, nil - } - for _, event := range events { + _ = req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } event.Results = req.makeResultEvent(event) - } - results = append(results, events...) + callback(event) + }) } - return results, nil + return nil } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 5eaf239f9..08fc283db 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -14,25 +14,9 @@ import ( // Match matches a generic data response again a given matcher func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { - partString := matcher.Part - switch partString { - case "header": - partString = "all_headers" - } - - var itemStr string - if partString == "all" { - builder := &strings.Builder{} - builder.WriteString(data["body"].(string)) - builder.WriteString("\n\n") - builder.WriteString(data["all_headers"].(string)) - itemStr = builder.String() - } else { - item, ok := data[partString] - if !ok { - return false - } - itemStr = types.ToString(item) + item, ok := getMatchPart(matcher.Part, data) + if !ok { + return false } switch matcher.GetType() { @@ -43,13 +27,13 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) } return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(item))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.Result(matcher.MatchWords(item)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.Result(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.Result(matcher.MatchBinary(item)) case matchers.DSLMatcher: return matcher.Result(matcher.MatchDSL(data)) } @@ -58,35 +42,41 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) // Extract performs extracting operation for a extractor on model and returns true or false. func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { - partString := extractor.Part - switch partString { - case "header": - partString = "all_headers" - } - - var itemStr string - if partString == "all" { - builder := &strings.Builder{} - builder.WriteString(data["body"].(string)) - builder.WriteString("\n\n") - builder.WriteString(data["all_headers"].(string)) - itemStr = builder.String() - } else { - item, ok := data[partString] - if !ok { - return nil - } - itemStr = types.ToString(item) + item, ok := getMatchPart(extractor.Part, data) + if !ok { + return nil } switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(item) case extractors.KValExtractor: return extractor.ExtractKval(data) } return nil } +// getMatchPart returns the match part honoring "all" matchers + others. +func getMatchPart(part string, data output.InternalEvent) (string, bool) { + if part == "header" { + part = "all_headers" + } + var itemStr string + + if part == "all" { + builder := &strings.Builder{} + builder.WriteString(data["body"].(string)) + builder.WriteString(data["all_headers"].(string)) + itemStr = builder.String() + } else { + item, ok := data[part] + if !ok { + return "", false + } + itemStr = types.ToString(item) + } + return itemStr, true +} + // responseToDSLMap converts a HTTP response to a map for use in DSL matching func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index c40d937c6..509bc3fa3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" @@ -27,7 +28,7 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { generator := e.newGenerator() maxWorkers := e.RaceNumberRequests @@ -35,7 +36,6 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int var requestErr error mutex := &sync.Mutex{} - var outputs []*output.InternalWrappedEvent for i := 0; i < e.RaceNumberRequests; i++ { request, err := generator.Make(reqURL, nil) if err != nil { @@ -44,23 +44,21 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int swg.Add() go func(httpRequest *generatedRequest) { - output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) - } else { - outputs = append(outputs, output...) } mutex.Unlock() swg.Done() }(request) } swg.Wait() - return outputs, requestErr + return requestErr } // executeRaceRequest executes race condition request for a URL -func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { generator := e.newGenerator() // Workers that keeps enqueuing new requests @@ -69,7 +67,6 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in var requestErr error mutex := &sync.Mutex{} - var outputs []*output.InternalWrappedEvent for { request, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { @@ -77,36 +74,34 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in } if err != nil { e.options.Progress.DecrementRequests(int64(generator.Total())) - return nil, err + return err } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() e.options.RateLimiter.Take() - output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) - } else { - outputs = append(outputs, output...) } mutex.Unlock() }(request) e.options.Progress.IncrementRequests() } swg.Wait() - return outputs, requestErr + return requestErr } // executeRaceRequest executes race condition request for a URL -func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { generator := e.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) if err != nil { - return nil, err + return err } pipeOptions := rawhttp.DefaultPipelineOptions @@ -130,7 +125,6 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter var requestErr error mutex := &sync.Mutex{} - var outputs []*output.InternalWrappedEvent for { request, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { @@ -138,7 +132,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter } if err != nil { e.options.Progress.DecrementRequests(int64(generator.Total())) - return nil, err + return err } request.pipelinedClient = pipeclient @@ -146,42 +140,39 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter go func(httpRequest *generatedRequest) { defer swg.Done() - output, err := e.executeRequest(reqURL, httpRequest, dynamicValues) + err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) - } else { - outputs = append(outputs, output...) } mutex.Unlock() }(request) e.options.Progress.IncrementRequests() } swg.Wait() - return outputs, requestErr + return requestErr } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { // verify if pipeline was requested if r.Pipeline { - return r.executeTurboHTTP(reqURL, dynamicValues) + return r.executeTurboHTTP(reqURL, dynamicValues, callback) } // verify if a basic race condition was requested if r.Race && r.RaceNumberRequests > 0 { - return r.executeRaceRequest(reqURL, dynamicValues) + return r.executeRaceRequest(reqURL, dynamicValues, callback) } // verify if parallel elaboration was requested if r.Threads > 0 { - return r.executeParallelHTTP(reqURL, dynamicValues) + return r.executeParallelHTTP(reqURL, dynamicValues, callback) } generator := r.newGenerator() var requestErr error - var outputs []*output.InternalWrappedEvent for { request, err := generator.Make(reqURL, dynamicValues) if err == io.EOF { @@ -189,34 +180,34 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int } if err != nil { r.options.Progress.DecrementRequests(int64(generator.Total())) - return nil, err + return err } + var gotOutput bool r.options.RateLimiter.Take() - output, err := r.executeRequest(reqURL, request, dynamicValues) + err = r.executeRequest(reqURL, request, dynamicValues, func(event *output.InternalWrappedEvent) { + // Add the extracts to the dynamic values if any. + if event.OperatorsResult != nil { + gotOutput = true + dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) + } + callback(event) + }) if err != nil { requestErr = multierr.Append(requestErr, err) - } else { - // Add the extracts to the dynamic values if any. - for _, o := range output { - if o.OperatorsResult != nil { - dynamicValues = generators.MergeMaps(dynamicValues, o.OperatorsResult.DynamicValues) - } - } - outputs = append(outputs, output...) } r.options.Progress.IncrementRequests() - if request.original.options.Options.StopAtFirstMatch && len(output) > 0 { + if request.original.options.Options.StopAtFirstMatch && gotOutput { r.options.Progress.DecrementRequests(int64(generator.Total())) break } } - return outputs, requestErr + return requestErr } // executeRequest executes the actual generated request and returns error if occured -func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) { +func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}, callback protocols.OutputEventCallback) error { // Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given if r.options.Options.RandomAgent { builder := &strings.Builder{} @@ -235,7 +226,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if r.options.Options.Debug || r.options.ProjectFile != nil { dumpedRequest, err = dump(request, reqURL) if err != nil { - return nil, err + return err } } if r.options.Options.Debug { @@ -277,7 +268,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) r.options.Progress.DecrementRequests(1) - return nil, err + return err } gologger.Verbose().Msgf("Sent request to %s", reqURL) r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) @@ -289,7 +280,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam var dumpErr error dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { - return nil, errors.Wrap(dumpErr, "could not dump http response") + return errors.Wrap(dumpErr, "could not dump http response") } } @@ -297,7 +288,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if err != nil { _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - return nil, errors.Wrap(err, "could not read http body") + return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -307,7 +298,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam dataOrig := data data, err = handleDecompression(request, data) if err != nil { - return nil, errors.Wrap(err, "could not decompress http body") + return errors.Wrap(err, "could not decompress http body") } // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) @@ -321,7 +312,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if r.options.ProjectFile != nil && !fromcache { err := r.options.ProjectFile.Set(dumpedRequest, resp, data) if err != nil { - return nil, errors.Wrap(err, "could not store in project file") + return errors.Wrap(err, "could not store in project file") } } @@ -352,18 +343,19 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if request.request != nil { matchedURL = request.request.URL.String() } - ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + ouputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) - event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { - return nil, nil + return nil } result.PayloadValues = request.meta - event[0].OperatorsResult = result + event.OperatorsResult = result + callback(event) } - return event, nil + return nil } const two = 2 diff --git a/v2/pkg/protocols/network/executer.go b/v2/pkg/protocols/network/executer.go index 52f59eb5e..2f1251970 100644 --- a/v2/pkg/protocols/network/executer.go +++ b/v2/pkg/protocols/network/executer.go @@ -43,48 +43,29 @@ func (e *Executer) Execute(input string) (bool, error) { var results bool for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return false, err - } - if events == nil { - return false, nil - } - - // If we have a result field, we should add a result to slice. - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } - for _, result := range req.makeResultEvent(event) { results = true e.options.Output.Write(result) } - } + }) } return results, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { - var results []*output.InternalWrappedEvent - +func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { for _, req := range e.requests { - events, err := req.ExecuteWithResults(input, nil) - if err != nil { - return nil, err - } - if events == nil { - return nil, nil - } - for _, event := range events { + _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { - continue + return } event.Results = req.makeResultEvent(event) - } - results = append(results, events...) + callback(event) + }) } - return results, nil + return nil } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index f33e43248..2389e3247 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -20,15 +20,14 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { address, err := getAddress(input) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not get address from url") + return errors.Wrap(err, "could not get address from url") } - var outputs []*output.InternalWrappedEvent for _, kv := range r.addresses { replacer := replacer.New(map[string]interface{}{"Hostname": address}) actualAddress := replacer.Replace(kv.key) @@ -39,30 +38,29 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent actualAddress = net.JoinHostPort(actualAddress, kv.value) } - output, err := r.executeAddress(actualAddress, address, input) + err = r.executeAddress(actualAddress, address, input, callback) if err != nil { gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } - outputs = append(outputs, output...) } - return outputs, nil + return nil } // executeAddress executes the request for an address -func (r *Request) executeAddress(actualAddress, address, input string) ([]*output.InternalWrappedEvent, error) { +func (r *Request) executeAddress(actualAddress, address, input string, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, err + return err } conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress) if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not connect to server request") + return errors.Wrap(err, "could not connect to server request") } defer conn.Close() conn.SetReadDeadline(time.Now().Add(5 * time.Second)) @@ -80,7 +78,7 @@ func (r *Request) executeAddress(actualAddress, address, input string) ([]*outpu if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not write request to server") + return errors.Wrap(err, "could not write request to server") } reqBuilder.Grow(len(input.Data)) reqBuilder.WriteString(input.Data) @@ -89,14 +87,14 @@ func (r *Request) executeAddress(actualAddress, address, input string) ([]*outpu if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not write request to server") + return errors.Wrap(err, "could not write request to server") } r.options.Progress.IncrementRequests() } if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) r.options.Progress.DecrementRequests(1) - return nil, errors.Wrap(err, "could not write request to server") + return errors.Wrap(err, "could not write request to server") } if r.options.Options.Debug { @@ -122,15 +120,16 @@ func (r *Request) executeAddress(actualAddress, address, input string) ([]*outpu } ouputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress) - event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) if !ok { - return nil, nil + return nil } - event[0].OperatorsResult = result + event.OperatorsResult = result } - return event, nil + callback(event) + return nil } // getAddress returns the address of the host to make request to diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 3293fc8e1..c808d658b 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -20,7 +20,7 @@ type Executer interface { // Execute executes the protocol group and returns true or false if results were found. Execute(input string) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) + ExecuteWithResults(input string, callback OutputEventCallback) error } // ExecuterOptions contains the configuration options for executer clients @@ -56,5 +56,8 @@ type Request interface { // Extract performs extracting operation for a extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) + ExecuteWithResults(input string, metadata output.InternalEvent, callback OutputEventCallback) error } + +// OutputEventCallback is a callback event for any results found during scanning. +type OutputEventCallback func(result *output.InternalWrappedEvent) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 7fa79d5fb..581e4c902 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -64,8 +64,11 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro } template.TotalRequests += template.Executer.Requests() - if err != nil { - return nil, errors.Wrap(err, "could not compile request") + if template.Executer != nil { + err := template.Executer.Compile() + if err != nil { + return nil, errors.Wrap(err, "could not compile request") + } } return template, nil } diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 9e74be30d..11ac6b9d6 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -1,6 +1,9 @@ package workflows -import "go.uber.org/atomic" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "go.uber.org/atomic" +) // RunWorkflow runs a workflow on an input and returns true or false func (w *Workflow) RunWorkflow(input string) (bool, error) { @@ -35,32 +38,32 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(template.Matchers) > 0 { w.options.Progress.AddToTotal(int64(template.Executer.Requests())) - output, err := template.Executer.ExecuteWithResults(input) - if err != nil { - return err - } - if len(output) == 0 { - return nil - } + var executionErr error + err := template.Executer.ExecuteWithResults(input, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult == nil { + return + } - for _, matcher := range template.Matchers { - for _, item := range output { - if item.OperatorsResult == nil { - continue - } - - _, matchOK := item.OperatorsResult.Matches[matcher.Name] - _, extractOK := item.OperatorsResult.Extracts[matcher.Name] + for _, matcher := range template.Matchers { + _, matchOK := event.OperatorsResult.Matches[matcher.Name] + _, extractOK := event.OperatorsResult.Extracts[matcher.Name] if !matchOK && !extractOK { continue } for _, subtemplate := range matcher.Subtemplates { if err := w.runWorkflowStep(subtemplate, input, results); err != nil { - return err + executionErr = err + break } } } + }) + if err != nil { + return err + } + if executionErr != nil { + return executionErr } return nil } diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index dfe1f4036..8bd345689 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/stretchr/testify/require" ) @@ -162,9 +163,12 @@ func (m *mockExecuter) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (m *mockExecuter) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { +func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { if m.executeHook != nil { m.executeHook(input) } - return m.outputs, nil + for _, output := range m.outputs { + callback(output) + } + return nil } From 8afd465c7899bbb981611d44595ecdf18e7d8dab Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 2 Jan 2021 02:39:27 +0530 Subject: [PATCH 59/92] Added a common executer package with request interfaces --- .../{http => common/executer}/executer.go | 9 ++- v2/pkg/protocols/dns/executer.go | 71 ------------------- v2/pkg/protocols/dns/request.go | 1 + v2/pkg/protocols/file/executer.go | 71 ------------------- v2/pkg/protocols/file/request.go | 1 + v2/pkg/protocols/http/request.go | 25 +------ v2/pkg/protocols/network/executer.go | 71 ------------------- v2/pkg/protocols/network/request.go | 1 + v2/pkg/templates/compile.go | 26 ++++--- 9 files changed, 28 insertions(+), 248 deletions(-) rename v2/pkg/protocols/{http => common/executer}/executer.go (89%) delete mode 100644 v2/pkg/protocols/dns/executer.go delete mode 100644 v2/pkg/protocols/file/executer.go delete mode 100644 v2/pkg/protocols/network/executer.go diff --git a/v2/pkg/protocols/http/executer.go b/v2/pkg/protocols/common/executer/executer.go similarity index 89% rename from v2/pkg/protocols/http/executer.go rename to v2/pkg/protocols/common/executer/executer.go index d12dad14e..09bccca99 100644 --- a/v2/pkg/protocols/http/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -1,4 +1,4 @@ -package http +package executer import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -7,14 +7,14 @@ import ( // Executer executes a group of requests for a protocol type Executer struct { - requests []*Request + requests []protocols.Request options *protocols.ExecuterOptions } var _ protocols.Executer = &Executer{} // NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { +func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOptions) *Executer { return &Executer{requests: requests, options: options} } @@ -48,7 +48,7 @@ func (e *Executer) Execute(input string) (bool, error) { if event.OperatorsResult == nil { return } - for _, result := range req.makeResultEvent(event) { + for _, result := range event.Results { results = true e.options.Output.Write(result) } @@ -68,7 +68,6 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve if event.OperatorsResult == nil { return } - event.Results = req.makeResultEvent(event) callback(event) }) } diff --git a/v2/pkg/protocols/dns/executer.go b/v2/pkg/protocols/dns/executer.go deleted file mode 100644 index 62216dc25..000000000 --- a/v2/pkg/protocols/dns/executer.go +++ /dev/null @@ -1,71 +0,0 @@ -package dns - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []*Request - options *protocols.ExecuterOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - for _, request := range e.requests { - err := request.Compile(e.options) - if err != nil { - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += request.Requests() - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { - var results bool - - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - for _, result := range req.makeResultEvent(event) { - results = true - e.options.Output.Write(result) - } - }) - } - return results, nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - event.Results = req.makeResultEvent(event) - callback(event) - }) - } - return nil -} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 90785f34f..37a64b9ff 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -61,6 +61,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent return nil } event.OperatorsResult = result + event.Results = r.makeResultEvent(event) } callback(event) return nil diff --git a/v2/pkg/protocols/file/executer.go b/v2/pkg/protocols/file/executer.go deleted file mode 100644 index 286f339de..000000000 --- a/v2/pkg/protocols/file/executer.go +++ /dev/null @@ -1,71 +0,0 @@ -package file - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []*Request - options *protocols.ExecuterOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - for _, request := range e.requests { - err := request.Compile(e.options) - if err != nil { - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += request.Requests() - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { - var results bool - - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - for _, result := range req.makeResultEvent(event) { - results = true - e.options.Output.Write(result) - } - }) - } - return results, nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - event.Results = req.makeResultEvent(event) - callback(event) - }) - } - return nil -} diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index cbcf63ef4..5b24bc16a 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -55,6 +55,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent return } event.OperatorsResult = result + event.Results = r.makeResultEvent(event) callback(event) } }) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 509bc3fa3..99111d7f3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -154,7 +154,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error { +func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested if r.Pipeline { return r.executeTurboHTTP(reqURL, dynamicValues, callback) @@ -316,26 +316,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } } - // store for internal purposes the DSL matcher data - // hardcode stopping storing data after defaultMaxHistorydata items - //if len(result.historyData) < defaultMaxHistorydata { - // result.Lock() - // // update history data with current reqURL and hostname - // result.historyData["reqURL"] = reqURL - // if parsed, err := url.Parse(reqURL); err == nil { - // result.historyData["Hostname"] = parsed.Host - // } - // result.historyData = generators.MergeMaps(result.historyData, matchers.HTTPToMap(resp, body, headers, duration, format)) - // if payloads == nil { - // // merge them to history data - // result.historyData = generators.MergeMaps(result.historyData, payloads) - // } - // result.historyData = generators.MergeMaps(result.historyData, dynamicvalues) - // - // // complement match data with new one if necessary - // matchData = generators.MergeMaps(matchData, result.historyData) - // result.Unlock() - //} var matchedURL string if request.rawRequest != nil { matchedURL = request.rawRequest.FullURL @@ -351,8 +331,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if !ok { return nil } - result.PayloadValues = request.meta event.OperatorsResult = result + result.PayloadValues = request.meta + event.Results = r.makeResultEvent(event) callback(event) } return nil diff --git a/v2/pkg/protocols/network/executer.go b/v2/pkg/protocols/network/executer.go deleted file mode 100644 index 2f1251970..000000000 --- a/v2/pkg/protocols/network/executer.go +++ /dev/null @@ -1,71 +0,0 @@ -package network - -import ( - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []*Request - options *protocols.ExecuterOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - for _, request := range e.requests { - err := request.Compile(e.options) - if err != nil { - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += int(request.Requests()) - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { - var results bool - - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - for _, result := range req.makeResultEvent(event) { - results = true - e.options.Output.Write(result) - } - }) - } - return results, nil -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { - for _, req := range e.requests { - _ = req.ExecuteWithResults(input, nil, func(event *output.InternalWrappedEvent) { - if event.OperatorsResult == nil { - return - } - event.Results = req.makeResultEvent(event) - callback(event) - }) - } - return nil -} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 2389e3247..34cbb1891 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -127,6 +127,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback return nil } event.OperatorsResult = result + event.Results = r.makeResultEvent(event) } callback(event) return nil diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 581e4c902..33b2e4bfe 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -6,10 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -50,17 +47,30 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro } // Compile the requests found + requests := []protocols.Request{} if len(template.RequestsDNS) > 0 { - template.Executer = dns.NewExecuter(template.RequestsDNS, options) + for _, req := range template.RequestsDNS { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, options) } if len(template.RequestsHTTP) > 0 { - template.Executer = http.NewExecuter(template.RequestsHTTP, options) + for _, req := range template.RequestsHTTP { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, options) } if len(template.RequestsFile) > 0 { - template.Executer = file.NewExecuter(template.RequestsFile, options) + for _, req := range template.RequestsFile { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, options) } if len(template.RequestsNetwork) > 0 { - template.Executer = network.NewExecuter(template.RequestsNetwork, options) + for _, req := range template.RequestsNetwork { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, options) } template.TotalRequests += template.Executer.Requests() From ff3b0e116d7d5d7cce12a22df5704832a459bfe9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 19:38:16 +0530 Subject: [PATCH 60/92] Fixed network protocol not working --- v2/pkg/protocols/network/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index f68f22822..a99f3e07a 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -80,5 +80,5 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Requests returns the total number of requests the YAML rule will perform func (r *Request) Requests() int { - return len(r.addresses) + return len(r.Address) } From 9c816801739d39c5b75bb378055ef3b42f9e3bfa Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 19:59:12 +0530 Subject: [PATCH 61/92] Fixed verbose output for sent requests --- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/http/request.go | 10 ++++++---- v2/pkg/protocols/network/request.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 5b24bc16a..6f4e467e6 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -45,7 +45,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) fmt.Fprintf(os.Stderr, "%s\n", dataStr) } - gologger.Verbose().Msgf("[%s] Sent file request to %s", r.options.TemplateID, data) + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) ouputEvent := r.responseToDSLMap(dataStr, input, data) event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 99111d7f3..0d1838588 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -234,12 +234,13 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } + var formedURL string timeStart := time.Now() if request.original.Pipeline { + formedURL = request.rawRequest.FullURL resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) } else if request.original.Unsafe { - // rawhttp - // burp uses "\r\n" as new line character + formedURL = request.rawRequest.FullURL request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n") options := request.original.rawhttpClient.Options options.AutomaticContentLength = !r.DisableAutoContentLength @@ -247,6 +248,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam options.FollowRedirects = r.Redirects resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) } else { + formedURL = request.request.URL.String() // if nuclei-project is available check if the request was already sent previously if r.options.ProjectFile != nil { // if unavailable fail silently @@ -270,7 +272,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam r.options.Progress.DecrementRequests(1) return err } - gologger.Verbose().Msgf("Sent request to %s", reqURL) + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL) r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) duration := time.Since(timeStart) @@ -304,7 +306,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) if r.options.Options.Debug { dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, reqURL) + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 34cbb1891..7832cec7d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -104,7 +104,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback } r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) - gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress) + gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 if r.ReadSize != 0 { From f92a37426c7a199fa61589d3d38004a3b9e0ccfb Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 20:09:55 +0530 Subject: [PATCH 62/92] Fixed panic with workflows --- v2/internal/runner/runner.go | 2 +- v2/pkg/templates/compile.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9c40f3819..7de3efb91 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -266,7 +266,7 @@ func (r *Runner) RunEnumeration() { r.output.Close() os.Remove(r.options.Output) } - gologger.Info().Msgf("No results found. Happy hacking!") + gologger.Info().Msgf("No results found. Maybe try again...!") } } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 33b2e4bfe..132385e94 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -72,9 +72,9 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro } template.Executer = executer.NewExecuter(requests, options) } - template.TotalRequests += template.Executer.Requests() if template.Executer != nil { + template.TotalRequests += template.Executer.Requests() err := template.Executer.Compile() if err != nil { return nil, errors.Wrap(err, "could not compile request") From 223122a7edbdc8987d38e67e984421e2bc190e10 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 20:15:08 +0530 Subject: [PATCH 63/92] Fixed double trailing slashes in inputs --- v2/pkg/protocols/http/build_request.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 23e79d2eb..0b8d21ff1 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -111,6 +111,9 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() + if strings.HasSuffix(baseURL, "/") { + baseURL = strings.TrimSuffix(baseURL, "/") + } parsed, err := url.Parse(baseURL) if err != nil { return nil, err From d9c6eb0147ee755a61e93c2c361eba4f996c3d75 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 20:19:16 +0530 Subject: [PATCH 64/92] No results found string update --- v2/internal/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 7de3efb91..3f536a5e7 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -266,7 +266,7 @@ func (r *Runner) RunEnumeration() { r.output.Close() os.Remove(r.options.Output) } - gologger.Info().Msgf("No results found. Maybe try again...!") + gologger.Info().Msgf("No results found. Better luck next time!") } } From f016893ee5f9a342131e1b91ad8a5b570c089391 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 20:57:37 +0530 Subject: [PATCH 65/92] Fixed longstanding update-templates bug with deletions not visible --- v2/internal/runner/update.go | 92 ++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 1c8d9f621..04fceba08 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -2,8 +2,11 @@ package runner import ( "archive/zip" + "bufio" "bytes" "context" + "crypto/md5" + "encoding/hex" "errors" "fmt" "io" @@ -54,7 +57,7 @@ func (r *Runner) updateTemplates() error { } // Use custom location if user has given a template directory - if r.options.TemplatesDirectory != "" { + if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { home = r.options.TemplatesDirectory } r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")} @@ -64,7 +67,7 @@ func (r *Runner) updateTemplates() error { if getErr != nil { return getErr } - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL()) if err != nil { @@ -209,6 +212,15 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string return fmt.Errorf("failed to create template base folder: %s", err) } + // We use file-checksums that are md5 hashes to store the list of files->hashes + // that have been downloaded previously. + // If the path isn't found in new update after being read from the previous checksum, + // it is removed. This allows us fine-grained control over the download process + // as well as solves a long problem with nuclei-template updates. + checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") + previousChecksum := readPreviousTemplatesChecksum(checksumFile) + + checksums := make(map[string]string) for _, file := range z.File { directory, name := filepath.Split(file.Name) if name == "" { @@ -224,7 +236,8 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string return fmt.Errorf("failed to create template folder %s : %s", templateDirectory, err) } - f, err := os.OpenFile(path.Join(templateDirectory, name), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) + templatePath := path.Join(templateDirectory, name) + f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) if err != nil { f.Close() return fmt.Errorf("could not create uncompressed file: %s", err) @@ -235,13 +248,84 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string f.Close() return fmt.Errorf("could not open archive to extract file: %s", err) } + hasher := md5.New() - _, err = io.Copy(f, reader) + // Save file and also read into hasher for md5 + _, err = io.Copy(f, io.TeeReader(reader, hasher)) if err != nil { f.Close() return fmt.Errorf("could not write template file: %s", err) } f.Close() + checksums[templatePath] = hex.EncodeToString(hasher.Sum(nil)) + } + + // If we don't find a previous file in new download and it hasn't been + // changed on the disk, delete it. + if previousChecksum != nil { + for k, v := range previousChecksum { + _, ok := checksums[k] + if !ok && v[0] == v[1] { + os.Remove(k) + } + } + } + return writeTemplatesChecksum(checksumFile, checksums) +} + +// readPreviousTemplatesChecksum reads the previous checksum file from the disk. +// +// It reads two checksums, the first checksum is what we expect and the second is +// the actual checksum of the file on disk currently. +func readPreviousTemplatesChecksum(file string) map[string][2]string { + f, err := os.Open(file) + if err != nil { + return nil + } + defer f.Close() + scanner := bufio.NewScanner(f) + + checksum := make(map[string][2]string) + for scanner.Scan() { + text := scanner.Text() + if text == "" { + continue + } + parts := strings.Split(text, ",") + if len(parts) < 2 { + continue + } + values := [2]string{parts[1]} + + f, err := os.Open(parts[0]) + if err != nil { + continue + } + defer f.Close() + + hasher := md5.New() + if _, err := io.Copy(hasher, f); err != nil { + continue + } + values[1] = hex.EncodeToString(hasher.Sum(nil)) + checksum[parts[0]] = values + } + return checksum +} + +// writeTemplatesChecksum writes the nuclei-templates checksum data to disk. +func writeTemplatesChecksum(file string, checksum map[string]string) error { + f, err := os.Create(file) + if err != nil { + return nil + } + defer f.Close() + + for k, v := range checksum { + f.WriteString(k) + f.WriteString(",") + f.WriteString(v) + f.WriteString("\n") } return nil } From 77817277a262f284c0f3b0701e89081bcde947c6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 11 Jan 2021 21:11:35 +0530 Subject: [PATCH 66/92] Added extractor name field --- v2/pkg/operators/operators.go | 2 +- v2/pkg/output/output.go | 2 ++ v2/pkg/protocols/dns/operators.go | 36 ++++++++++++++++++--------- v2/pkg/protocols/file/operators.go | 36 ++++++++++++++++++--------- v2/pkg/protocols/http/operators.go | 36 ++++++++++++++++++--------- v2/pkg/protocols/network/operators.go | 36 ++++++++++++++++++--------- 6 files changed, 99 insertions(+), 49 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 9e9f13375..818cba737 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -110,7 +110,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac result.OutputExtracts = append(result.OutputExtracts, match) } } - if len(extractorResults) > 0 && !extractor.Internal { + if len(extractorResults) > 0 && !extractor.Internal && extractor.Name != "" { result.Extracts[extractor.Name] = extractorResults } } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 2faab9a62..4c5e5db4f 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -56,6 +56,8 @@ type ResultEvent struct { Info map[string]string `json:"info"` // MatcherName is the name of the matcher matched if any. MatcherName string `json:"matcher_name,omitempty"` + // ExtractorName is the name of the extractor matched if any. + ExtractorName string `json:"extractor_name,omitempty"` // Type is the type of the result event. Type string `json:"type"` // Host is the host input on which match was found. diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 88c1aed2e..d11476be7 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -120,7 +120,29 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - data := output.ResultEvent{ + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ TemplateID: r.options.TemplateID, Info: r.options.TemplateInfo, Type: "dns", @@ -132,15 +154,5 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu data.Request = wrapped.InternalEvent["request"].(string) data.Response = wrapped.InternalEvent["raw"].(string) } - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data.MatcherName = k - results = append(results, &data) - } - } else { - results = append(results, &data) - } - return results + return data } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 894f833e4..a8262360a 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -79,7 +79,29 @@ func (r *Request) responseToDSLMap(raw string, host, matched string) output.Inte func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - data := output.ResultEvent{ + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ TemplateID: r.options.TemplateID, Info: r.options.TemplateInfo, Type: "file", @@ -90,15 +112,5 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu if r.options.Options.JSONRequests { data.Response = wrapped.InternalEvent["raw"].(string) } - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data.MatcherName = k - results = append(results, &data) - } - } else { - results = append(results, &data) - } - return results + return data } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 08fc283db..470c3c589 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -116,7 +116,29 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - data := output.ResultEvent{ + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ TemplateID: r.options.TemplateID, Info: r.options.TemplateInfo, Type: "http", @@ -129,15 +151,5 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu data.Request = wrapped.InternalEvent["request"].(string) data.Response = wrapped.InternalEvent["raw"].(string) } - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data.MatcherName = k - results = append(results, &data) - } - } else { - results = append(results, &data) - } - return results + return data } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 8d85ee196..c2748f739 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -82,7 +82,29 @@ func (r *Request) responseToDSLMap(req, resp string, host, matched string) outpu func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - data := output.ResultEvent{ + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ TemplateID: r.options.TemplateID, Info: r.options.TemplateInfo, Type: "network", @@ -94,15 +116,5 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu data.Request = wrapped.InternalEvent["request"].(string) data.Response = wrapped.InternalEvent["data"].(string) } - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data.MatcherName = k - results = append(results, &data) - } - } else { - results = append(results, &data) - } - return results + return data } From d4191814c7d198eec17beb198120be8eb398a12b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 00:49:26 +0530 Subject: [PATCH 67/92] Fixed race condition requests not being sent --- v2/pkg/protocols/http/request.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0d1838588..38d4653f6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -36,12 +36,12 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int var requestErr error mutex := &sync.Mutex{} - for i := 0; i < e.RaceNumberRequests; i++ { - request, err := generator.Make(reqURL, nil) - if err != nil { - break - } + request, err := generator.Make(reqURL, nil) + if err != nil { + return err + } + for i := 0; i < e.RaceNumberRequests; i++ { swg.Add() go func(httpRequest *generatedRequest) { err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback) From 4d800d8c0c6666a32e1bcf60686dead78539ee6a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 02:00:11 +0530 Subject: [PATCH 68/92] Fixed bugs with progress and http path / handling --- v2/pkg/protocols/common/executer/executer.go | 2 +- v2/pkg/protocols/http/build_request.go | 20 +++++++++----------- v2/pkg/protocols/http/http.go | 19 +++++++++++++++---- v2/pkg/protocols/http/raw/raw.go | 7 +++---- v2/pkg/protocols/http/request.go | 4 ++-- v2/pkg/templates/compile.go | 2 +- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 09bccca99..12f8a3653 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -33,7 +33,7 @@ func (e *Executer) Compile() error { func (e *Executer) Requests() int { var count int for _, request := range e.requests { - count += int(request.Requests()) + count += request.Requests() } return count } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 0b8d21ff1..b2c515bad 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -111,9 +111,6 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() - if strings.HasSuffix(baseURL, "/") { - baseURL = strings.TrimSuffix(baseURL, "/") - } parsed, err := url.Parse(baseURL) if err != nil { return nil, err @@ -158,6 +155,9 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { // MakeHTTPRequestFromModel creates a *http.Request from a request template func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*generatedRequest, error) { + if strings.HasSuffix(values["BaseURL"].(string), "/") { + data = strings.TrimPrefix(data, "/") + } URL := replacer.New(values).Replace(data) // Build a request on the specified URL @@ -190,24 +190,22 @@ func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, } // handleRawWithPaylods handles raw requests along with paylaods -func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, genValues map[string]interface{}) (*generatedRequest, error) { +func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, baseURL string, values, generatorValues map[string]interface{}) (*generatedRequest, error) { baseValues := generators.CopyMap(values) - finValues := generators.MergeMaps(baseValues, genValues) + finalValues := generators.MergeMaps(baseValues, generatorValues) // Replace the dynamic variables in the URL if any - rawRequest = replacer.New(finValues).Replace(rawRequest) + rawRequest = replacer.New(finalValues).Replace(rawRequest) dynamicValues := make(map[string]interface{}) for _, match := range templateExpressionRegex.FindAllString(rawRequest, -1) { // check if the match contains a dynamic variable expr := generators.TrimDelimiters(match) compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { return nil, err } - - result, err := compiled.Evaluate(finValues) + result, err := compiled.Evaluate(finalValues) if err != nil { return nil, err } @@ -223,7 +221,7 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, // rawhttp if r.request.Unsafe { - unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: genValues, original: r.request} + unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request} return unsafeReq, nil } @@ -250,7 +248,7 @@ func (r *requestGenerator) handleRawWithPaylods(ctx context.Context, rawRequest, if err != nil { return nil, err } - return &generatedRequest{request: request, meta: genValues, original: r.request}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil } // fillRequest fills various headers in the request with values diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 40c8757bb..d1e714766 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -114,11 +114,22 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { // Requests returns the total number of requests the YAML rule will perform func (r *Request) Requests() int { if r.generator != nil { - payloadRequests := r.generator.NewIterator().Total() - return len(r.Raw) * payloadRequests + payloadRequests := r.generator.NewIterator().Total() * len(r.Raw) + if r.Threads != 0 { + payloadRequests = payloadRequests * r.Threads + } + return payloadRequests } if len(r.Raw) > 0 { - return len(r.Raw) + requests := len(r.Raw) + if r.Threads != 0 { + requests = requests * r.Threads + } + return requests } - return len(r.Path) + requests := len(r.Path) + if r.Threads != 0 { + requests = requests * r.Threads + } + return requests } diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index c14bf3999..53401f55e 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -94,12 +94,11 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if rawRequest.Path == "" { rawRequest.Path = parsedURL.Path } else if strings.HasPrefix(rawRequest.Path, "?") { - // requests generated from http.ReadRequest have incorrect RequestURI, so they - // cannot be used to perform another request directly, we need to generate a new one - // with the new target url rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) } - + if strings.HasSuffix(baseURL, "/") { + rawRequest.Path = strings.TrimPrefix(rawRequest.Path, "/") + } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) // Set the request body diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 38d4653f6..903d0839d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -264,8 +264,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } } if err != nil { - if resp != nil { - _, _ = io.Copy(ioutil.Discard, resp.Body) + if resp != nil && resp.Body != nil { + // _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 132385e94..6d77e40c9 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -74,11 +74,11 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro } if template.Executer != nil { - template.TotalRequests += template.Executer.Requests() err := template.Executer.Compile() if err != nil { return nil, errors.Wrap(err, "could not compile request") } + template.TotalRequests += template.Executer.Requests() } return template, nil } From c029b8e6e732a347c9cc032379b33ace6a0b2195 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 02:05:41 +0530 Subject: [PATCH 69/92] Fixed panic in pipelined requests --- v2/pkg/protocols/http/request.go | 5 +++-- v2/pkg/templates/compile.go | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 903d0839d..c4fba94c6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -264,8 +264,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } } if err != nil { - if resp != nil && resp.Body != nil { - // _, _ = io.Copy(ioutil.Discard, resp.Body) + // rawhttp doesn't supports draining response bodies. + if resp != nil && resp.Body != nil && request.rawRequest == nil { + _, _ = io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() } r.options.Output.Request(r.options.TemplateID, reqURL, "http", err) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 6d77e40c9..a63ad4770 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -72,7 +72,6 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro } template.Executer = executer.NewExecuter(requests, options) } - if template.Executer != nil { err := template.Executer.Compile() if err != nil { From acb4d270ca6f762ec8decfffa977501908de63b7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 02:26:19 +0530 Subject: [PATCH 70/92] Misc fixes --- v2/internal/runner/runner.go | 20 ++------------------ v2/pkg/protocols/http/raw/raw.go | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 3f536a5e7..738f5a335 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -14,9 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -178,7 +176,7 @@ func (r *Runner) Close() { // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { - err := r.initializeProtocols() + err := protocolinit.Init(r.options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) } @@ -269,17 +267,3 @@ func (r *Runner) RunEnumeration() { gologger.Info().Msgf("No results found. Better luck next time!") } } - -// initializeProtocols initializes all the protocols and their caches -func (r *Runner) initializeProtocols() error { - if err := dnsclientpool.Init(r.options); err != nil { - return err - } - if err := httpclientpool.Init(r.options); err != nil { - return err - } - if err := networkclientpool.Init(r.options); err != nil { - return err - } - return nil -} diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 53401f55e..000c21eef 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -96,7 +96,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } else if strings.HasPrefix(rawRequest.Path, "?") { rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) } - if strings.HasSuffix(baseURL, "/") { + if strings.HasSuffix(hostURL, "/") { rawRequest.Path = strings.TrimPrefix(rawRequest.Path, "/") } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) From a15e0b75235ef4ee0d96a3f169c0f174e8717d8f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 02:27:32 +0530 Subject: [PATCH 71/92] Misc --- v2/pkg/protocols/common/protocolinit/init.go | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 v2/pkg/protocols/common/protocolinit/init.go diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go new file mode 100644 index 000000000..db024ceaa --- /dev/null +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -0,0 +1,22 @@ +package protocolinit + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Init initializes the client pools for the protocols +func Init(options *types.Options) error { + if err := dnsclientpool.Init(options); err != nil { + return err + } + if err := httpclientpool.Init(options); err != nil { + return err + } + if err := networkclientpool.Init(options); err != nil { + return err + } + return nil +} From 8110e60164ddb01be386f992816791c0cba68a84 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 02:44:51 +0530 Subject: [PATCH 72/92] Fixed payload files not working --- v2/pkg/protocols/http/http.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d1e714766..f2a5d18b3 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,6 +1,8 @@ package http import ( + "strings" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -101,6 +103,22 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.attackType = generators.StringToType[attackType] + // Resolve payload paths if they are files. + for name, payload := range r.Payloads { + switch pt := payload.(type) { + case string: + elements := strings.Split(pt, "\n") + //golint:gomnd // this is not a magic number + if len(elements) < 2 { + final, err := options.Catalogue.ResolvePath(elements[0], options.TemplatePath) + if err != nil { + return errors.Wrap(err, "could not read payload file") + } + r.Payloads[name] = []interface{}{final} + } + } + } + r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") From 50eafb29d1cff2c6659b95e50c9810c6d95f96f6 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 11:21:32 +0530 Subject: [PATCH 73/92] Bugfix: kval extractor not working --- v2/pkg/operators/extractors/compile.go | 5 +++++ v2/pkg/operators/extractors/extract.go | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index 2a26a6a44..ceea5a1e3 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -3,6 +3,7 @@ package extractors import ( "fmt" "regexp" + "strings" ) // CompileExtractors performs the initial setup operation on a extractor @@ -24,6 +25,10 @@ func (e *Extractor) CompileExtractors() error { e.regexCompiled = append(e.regexCompiled, compiled) } + for i, kval := range e.KVal { + e.KVal[i] = strings.ToLower(kval) + } + // Setup the part of the request to match, if any. if e.Part == "" { e.Part = "body" diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 31c9aa6f2..5fecdcf9f 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,6 +1,8 @@ package extractors -import "github.com/projectdiscovery/nuclei/v2/pkg/types" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) // ExtractRegex extracts text from a corpus and returns it func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { From 0023aaed77e9f7c8e21d9f3dc4f7891fd02330ee Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 13:20:46 +0530 Subject: [PATCH 74/92] Misc bug fixes --- v2/pkg/protocols/common/generators/load.go | 2 ++ v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/http/operators.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 0c44b613c..8c449e81e 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,6 +2,7 @@ package generators import ( "bufio" + "fmt" "io" "os" "strings" @@ -29,6 +30,7 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) loadedPayloads[name] = payloads } case interface{}: + fmt.Printf("%v elements\n", pt) loadedPayloads[name] = cast.ToStringSlice(pt) } } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index f2a5d18b3..84018c73d 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -114,7 +114,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not read payload file") } - r.Payloads[name] = []interface{}{final} + r.Payloads[name] = final } } } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 470c3c589..f9462fd42 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -96,7 +96,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["body"] = body for _, cookie := range resp.Cookies() { - data[cookie.Name] = cookie.Value + data[strings.ToLower(cookie.Name)] = cookie.Value } for k, v := range resp.Header { k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_"))) From 3ee742816673c06a5b316276326f0efd93ebb96c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 15:14:49 +0530 Subject: [PATCH 75/92] Added initial config file support with cobra cli --- v2/cmd/nuclei/main.go | 110 +++++++++++++++++++-- v2/internal/runner/options.go | 49 +-------- v2/internal/runner/templates.go | 8 +- v2/pkg/protocols/common/generators/load.go | 2 - v2/pkg/protocols/http/http.go | 3 +- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/types/types.go | 32 +----- 7 files changed, 110 insertions(+), 96 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 66333e821..197e4dd9d 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -1,19 +1,109 @@ package main import ( + "os" + "path" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/spf13/cast" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +var ( + cfgFile string + + options = &types.Options{} + rootCmd = &cobra.Command{ + Use: "nuclei", + Short: "Nuclei is a fast and extensible security scanner", + Long: `Nuclei is a fast tool for configurable targeted scanning +based on templates offering massive extensibility and ease of use.`, + Run: func(cmd *cobra.Command, args []string) { + mergeViperConfiguration(cmd) + + runner.ParseOptions(options) + + nucleiRunner, err := runner.New(options) + if err != nil { + gologger.Fatal().Msgf("Could not create runner: %s\n", err) + } + + nucleiRunner.RunEnumeration() + nucleiRunner.Close() + }, + } ) func main() { - // Parse the command line flags and read config files - options := runner.ParseOptions() - - nucleiRunner, err := runner.New(options) - if err != nil { - gologger.Fatal().Msgf("Could not create runner: %s\n", err) - } - - nucleiRunner.RunEnumeration() - nucleiRunner.Close() + rootCmd.Execute() +} + +// mergeViperConfiguration merges the flag configuration with viper file. +func mergeViperConfiguration(cmd *cobra.Command) { + cmd.PersistentFlags().VisitAll(func(f *pflag.Flag) { + if !f.Changed && viper.IsSet(f.Name) { + switch p := viper.Get(f.Name).(type) { + case []interface{}: + for _, item := range p { + cmd.PersistentFlags().Set(f.Name, cast.ToString(item)) + } + default: + cmd.PersistentFlags().Set(f.Name, viper.GetString(f.Name)) + } + } + }) +} + +func init() { + home, _ := os.UserHomeDir() + templatesDirectory := path.Join(home, "nuclei-templates") + + cobra.OnInitialize(func() { + if cfgFile != "" { + viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err != nil { + gologger.Fatal().Msgf("Could not read config: %s\n", err) + } + } + }) + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "Nuclei config file (default is $HOME/.nuclei.yaml)") + rootCmd.PersistentFlags().BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") + rootCmd.PersistentFlags().IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") + rootCmd.PersistentFlags().StringVar(&options.Target, "target", "", "Target is a single target to scan using template") + rootCmd.PersistentFlags().StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") + rootCmd.PersistentFlags().StringSliceVar(&options.ExcludedTemplates, "exclude", []string{}, "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") + rootCmd.PersistentFlags().StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.") + rootCmd.PersistentFlags().StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on") + rootCmd.PersistentFlags().StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)") + rootCmd.PersistentFlags().StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server") + rootCmd.PersistentFlags().StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server") + rootCmd.PersistentFlags().BoolVar(&options.Silent, "silent", false, "Show only results in output") + rootCmd.PersistentFlags().BoolVar(&options.Version, "version", false, "Show version of nuclei") + rootCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "v", false, "Show Verbose output") + rootCmd.PersistentFlags().BoolVar(&options.NoColor, "no-color", false, "Disable colors in output") + rootCmd.PersistentFlags().IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") + rootCmd.PersistentFlags().IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") + rootCmd.PersistentFlags().BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value") + rootCmd.PersistentFlags().StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") + rootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") + rootCmd.PersistentFlags().BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") + rootCmd.PersistentFlags().StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") + rootCmd.PersistentFlags().StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates") + rootCmd.PersistentFlags().BoolVar(&options.JSON, "json", false, "Write json output to files") + rootCmd.PersistentFlags().BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output") + rootCmd.PersistentFlags().BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") + rootCmd.PersistentFlags().BoolVar(&options.TemplateList, "tl", false, "List available templates") + rootCmd.PersistentFlags().IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second") + rootCmd.PersistentFlags().BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)") + rootCmd.PersistentFlags().IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template") + rootCmd.PersistentFlags().IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel") + rootCmd.PersistentFlags().BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times") + rootCmd.PersistentFlags().StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled") + rootCmd.PersistentFlags().BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches") + rootCmd.PersistentFlags().BoolVar(&options.TemplatesVersion, "templates-version", false, "Shows the installed nuclei-templates version") + rootCmd.PersistentFlags().StringVar(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "", "Burp Collaborator BIID") } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 4134cb6b1..bf50a5ac5 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -2,10 +2,8 @@ package runner import ( "errors" - "flag" "net/url" "os" - "path" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -14,51 +12,7 @@ import ( ) // ParseOptions parses the command line flags provided by a user -func ParseOptions() *types.Options { - options := &types.Options{} - - home, _ := os.UserHomeDir() - templatesDirectory := path.Join(home, "nuclei-templates") - - flag.BoolVar(&options.Sandbox, "sandbox", false, "Run workflows in isolated sandbox mode") - flag.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") - flag.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") - flag.IntVar(&options.MaxWorkflowDuration, "workflow-duration", 10, "Max time for workflow run on single URL in minutes") - flag.StringVar(&options.Target, "target", "", "Target is a single target to scan using template") - flag.Var(&options.Templates, "t", "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") - flag.Var(&options.ExcludedTemplates, "exclude", "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") - flag.StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.") - flag.StringVar(&options.Targets, "l", "", "List of URLs to run templates on") - flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") - flag.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server") - flag.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server") - flag.BoolVar(&options.Silent, "silent", false, "Show only results in output") - flag.BoolVar(&options.Version, "version", false, "Show version of nuclei") - flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") - flag.BoolVar(&options.NoColor, "no-color", false, "Disable colors in output") - flag.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") - flag.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value") - flag.Var(&options.CustomHeaders, "H", "Custom Header.") - flag.BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") - flag.BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") - flag.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") - flag.StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates") - flag.BoolVar(&options.JSON, "json", false, "Write json output to files") - flag.BoolVar(&options.JSONRequests, "include-rr", false, "Write requests/responses for matches in JSON output") - flag.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") - flag.BoolVar(&options.TemplateList, "tl", false, "List available templates") - flag.IntVar(&options.RateLimit, "rate-limit", 150, "Rate-Limit (maximum requests/second") - flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)") - flag.IntVar(&options.BulkSize, "bulk-size", 25, "Maximum Number of hosts analyzed in parallel per template") - flag.IntVar(&options.TemplateThreads, "c", 10, "Maximum Number of templates executed in parallel") - flag.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times") - flag.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled") - flag.BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches") - flag.BoolVar(&options.TemplatesVersion, "templates-version", false, "Shows the installed nuclei-templates version") - flag.StringVar(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "", "Burp Collaborator BIID") - flag.Parse() - +func ParseOptions(options *types.Options) { // Check if stdin pipe was given options.Stdin = hasStdin() @@ -87,7 +41,6 @@ func ParseOptions() *types.Options { if err != nil { gologger.Fatal().Msgf("Program exiting: %s\n", err) } - return options } // hasStdin returns true if we have stdin input diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index c37506a6b..50e39cf32 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -13,10 +13,8 @@ import ( // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. -func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string) (parsedTemplates []*templates.Template, workflowCount int) { +func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []string) (parsedTemplates []*templates.Template, workflowCount int) { workflowCount = 0 - severities = strings.ToLower(severities) - allSeverities := strings.Split(severities, ",") filterBySeverity := len(severities) > 0 gologger.Info().Msgf("Loading templates...") @@ -31,7 +29,7 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities string workflowCount++ } sev := strings.ToLower(t.Info["severity"]) - if !filterBySeverity || hasMatchingSeverity(sev, allSeverities) { + if !filterBySeverity || hasMatchingSeverity(sev, severities) { parsedTemplates = append(parsedTemplates, t) gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } else { @@ -113,11 +111,11 @@ func (r *Runner) listAvailableTemplates() { func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool { for _, s := range allowedSeverities { + s = strings.ToLower(s) if s != "" && strings.HasPrefix(templateSeverity, s) { return true } } - return false } diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 8c449e81e..0c44b613c 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,7 +2,6 @@ package generators import ( "bufio" - "fmt" "io" "os" "strings" @@ -30,7 +29,6 @@ func loadPayloads(payloads map[string]interface{}) (map[string][]string, error) loadedPayloads[name] = payloads } case interface{}: - fmt.Printf("%v elements\n", pt) loadedPayloads[name] = cast.ToStringSlice(pt) } } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 84018c73d..3953cb75b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -8,7 +8,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -66,7 +65,7 @@ type Request struct { options *protocols.ExecuterOptions attackType generators.Type totalRequests int - customHeaders types.StringSlice + customHeaders []string generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index c4fba94c6..19955de40 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -213,7 +213,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam builder := &strings.Builder{} builder.WriteString("User-Agent: ") builder.WriteString(uarand.GetRandom()) - r.customHeaders.Set(builder.String()) + r.customHeaders = append(r.customHeaders, builder.String()) } r.setCustomHeaders(request) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 2744cbf5a..57e122d52 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -1,17 +1,11 @@ package types -import ( - "strings" -) - // Options contains the configuration options for nuclei scanner. type Options struct { // RandomAgent generates random User-Agent RandomAgent bool // Metrics enables display of metrics via an http endpoint Metrics bool - // Sandbox mode allows users to run isolated workflows with system commands disabled - Sandbox bool // Debug mode allows debugging request/responses for the engine Debug bool // Silent suppresses any extra text and only writes found URLs on screen. @@ -44,8 +38,6 @@ type Options struct { Project bool // MetricsPort is the port to show metrics on MetricsPort int - // MaxWorkflowDuration is the maximum time a workflow can run for a URL - MaxWorkflowDuration int // BulkSize is the of targets analyzed in parallel for each template BulkSize int // TemplateThreads is the number of templates executed in parallel @@ -56,14 +48,12 @@ type Options struct { Retries int // Rate-Limit is the maximum number of requests per specified target RateLimit int - // Thread controls the number of concurrent requests to make. - Threads int // BurpCollaboratorBiid is the Burp Collaborator BIID for polling interactions. BurpCollaboratorBiid string // ProjectPath allows nuclei to use a user defined project folder ProjectPath string // Severity filters templates based on their severity and only run the matching ones. - Severity string + Severity []string // Target is a single URL/Domain to scan using a template Target string // Targets specifies the targets to scan using templates. @@ -79,23 +69,9 @@ type Options struct { // TraceLogFile specifies a file to write with the trace of all requests TraceLogFile string // Templates specifies the template/templates to use - Templates StringSlice + Templates []string // ExcludedTemplates specifies the template/templates to exclude - ExcludedTemplates StringSlice + ExcludedTemplates []string // CustomHeaders is the list of custom global headers to send with each request. - CustomHeaders StringSlice -} - -// StringSlice is a slice of strings as input -type StringSlice []string - -// String returns the stringified version of string slice -func (s *StringSlice) String() string { - return strings.Join(*s, ",") -} - -// Set appends a value to the string slice -func (s *StringSlice) Set(value string) error { - *s = append(*s, value) - return nil + CustomHeaders []string } From ab2bb0226f8f7d593d31a5912f0e3f8023b26b9c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 12 Jan 2021 17:18:08 +0530 Subject: [PATCH 76/92] Debug req/resp mode support --- v2/cmd/nuclei/main.go | 4 +++- v2/pkg/protocols/dns/request.go | 4 ++-- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/http/request.go | 8 ++++---- v2/pkg/protocols/network/request.go | 4 ++-- v2/pkg/types/types.go | 4 ++++ 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 197e4dd9d..220024cad 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -76,7 +76,7 @@ func init() { rootCmd.PersistentFlags().StringVar(&options.Target, "target", "", "Target is a single target to scan using template") rootCmd.PersistentFlags().StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Template input dir/file/files to run on host. Can be used multiple times. Supports globbing.") rootCmd.PersistentFlags().StringSliceVar(&options.ExcludedTemplates, "exclude", []string{}, "Template input dir/file/files to exclude. Can be used multiple times. Supports globbing.") - rootCmd.PersistentFlags().StringVar(&options.Severity, "severity", "", "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.") + rootCmd.PersistentFlags().StringSliceVar(&options.Severity, "severity", []string{}, "Filter templates based on their severity and only run the matching ones. Comma-separated values can be used to specify multiple severities.") rootCmd.PersistentFlags().StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on") rootCmd.PersistentFlags().StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)") rootCmd.PersistentFlags().StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server") @@ -90,6 +90,8 @@ func init() { rootCmd.PersistentFlags().BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value") rootCmd.PersistentFlags().StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") rootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Allow debugging of request/responses") + rootCmd.PersistentFlags().BoolVar(&options.DebugRequests, "debug-req", false, "Allow debugging of request") + rootCmd.PersistentFlags().BoolVar(&options.DebugResponse, "debug-resp", false, "Allow debugging of response") rootCmd.PersistentFlags().BoolVar(&options.UpdateTemplates, "update-templates", false, "Update Templates updates the installed templates (optional)") rootCmd.PersistentFlags().StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") rootCmd.PersistentFlags().StringVar(&options.TemplatesDirectory, "update-directory", templatesDirectory, "Directory to use for storing nuclei-templates") diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 37a64b9ff..974d55ab8 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -31,7 +31,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent return errors.Wrap(err, "could not build request") } - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String()) } @@ -48,7 +48,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent r.options.Output.Request(r.options.TemplateID, domain, "dns", err) gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 6f4e467e6..b3461b976 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -41,7 +41,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent } dataStr := tostring.UnsafeToString(buffer) - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) fmt.Fprintf(os.Stderr, "%s\n", dataStr) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 19955de40..e1118792d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -223,13 +223,13 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam dumpedRequest []byte fromcache bool ) - if r.options.Options.Debug || r.options.ProjectFile != nil { + if r.options.Options.Debug || r.options.ProjectFile != nil || r.options.Options.DebugRequests { dumpedRequest, err = dump(request, reqURL) if err != nil { return err } } - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } @@ -279,7 +279,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam duration := time.Since(timeStart) // Dump response - Step 1 - Decompression not yet handled var dumpedResponse []byte - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugResponse { var dumpErr error dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { @@ -305,7 +305,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugResponse { dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 7832cec7d..23c676d2c 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -97,7 +97,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback return errors.Wrap(err, "could not write request to server") } - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) fmt.Fprintf(os.Stderr, "%s\n", reqBuilder.String()) @@ -114,7 +114,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback n, _ := conn.Read(buffer) resp := string(buffer[:n]) - if r.options.Options.Debug { + if r.options.Options.Debug || r.options.Options.DebugResponse { gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) fmt.Fprintf(os.Stderr, "%s\n", resp) } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 57e122d52..a60df6af3 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -8,6 +8,10 @@ type Options struct { Metrics bool // Debug mode allows debugging request/responses for the engine Debug bool + // DebugRequests mode allows debugging request for the engine + DebugRequests bool + // DebugResponse mode allows debugging response for the engine + DebugResponse bool // Silent suppresses any extra text and only writes found URLs on screen. Silent bool // Version specifies if we should just show version and exit From 02822a17c0afbf8b0240f3ec99afeb53055083c7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 13 Jan 2021 03:17:07 +0530 Subject: [PATCH 77/92] Added simplehttp-only clustering impl (wip) --- .../protocols/common/clusterer/clusterer.go | 49 ++++++++++++ .../common/clusterer/clusterer_test.go | 77 +++++++++++++++++++ v2/pkg/protocols/common/compare/compare.go | 37 +++++++++ v2/pkg/protocols/common/executer/executer.go | 5 ++ v2/pkg/protocols/http/cluster.go | 29 +++++++ 5 files changed, 197 insertions(+) create mode 100644 v2/pkg/protocols/common/clusterer/clusterer.go create mode 100644 v2/pkg/protocols/common/clusterer/clusterer_test.go create mode 100644 v2/pkg/protocols/common/compare/compare.go create mode 100644 v2/pkg/protocols/http/cluster.go diff --git a/v2/pkg/protocols/common/clusterer/clusterer.go b/v2/pkg/protocols/common/clusterer/clusterer.go new file mode 100644 index 000000000..29c1a309e --- /dev/null +++ b/v2/pkg/protocols/common/clusterer/clusterer.go @@ -0,0 +1,49 @@ +package clusterer + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/templates" +) + +// Cluster clusters a list of templates into a lesser number if possible based +// on the similarity between the sent requests. +// +// If the attributes match, multiple requests can be clustered into a single +// request which saves time and network resources during execution. +func Cluster(list map[string]*templates.Template) [][]*templates.Template { + final := [][]*templates.Template{} + + // Each protocol that can be clustered should be handled here. + for key, template := range list { + // We only cluster http requests as of now. + // Take care of requests that can't be clustered first. + if len(template.RequestsHTTP) == 0 { + delete(list, key) + final = append(final, []*templates.Template{template}) + continue + } + + delete(list, key) // delete element first so it's not found later. + // Find any/all similar matching request that is identical to + // this one and cluster them together for http protocol only. + if len(template.RequestsHTTP) == 1 { + cluster := []*templates.Template{} + + for otherKey, other := range list { + if len(other.RequestsHTTP) == 0 { + continue + } + if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) { + delete(list, otherKey) + cluster = append(cluster, other) + } + } + if len(cluster) > 0 { + cluster = append(cluster, template) + final = append(final, cluster) + continue + } + } + final = append(final, []*templates.Template{template}) + } + return final +} diff --git a/v2/pkg/protocols/common/clusterer/clusterer_test.go b/v2/pkg/protocols/common/clusterer/clusterer_test.go new file mode 100644 index 000000000..1622b12fd --- /dev/null +++ b/v2/pkg/protocols/common/clusterer/clusterer_test.go @@ -0,0 +1,77 @@ +package clusterer + +import ( + "fmt" + "testing" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestHTTPRequestsCluster(t *testing.T) { + catalogue := catalogue.New("/Users/ice3man/nuclei-templates") + templatesList, err := catalogue.GetTemplatePath("/Users/ice3man/nuclei-templates") + require.Nil(t, err, "could not get templates") + + protocolinit.Init(&types.Options{}) + list := make(map[string]*templates.Template) + for _, template := range templatesList { + executerOpts := &protocols.ExecuterOptions{ + Output: &mockOutput{}, + Options: &types.Options{}, + Progress: nil, + Catalogue: catalogue, + RateLimiter: nil, + ProjectFile: nil, + } + t, err := templates.Parse(template, executerOpts) + if err != nil { + continue + } + if _, ok := list[t.ID]; !ok { + list[t.ID] = t + } else { + // log.Fatalf("Duplicate template found: %v\n", t) + } + } + + totalClusterCount := 0 + totalRequestsSentNew := 0 + new := Cluster(list) + for i, cluster := range new { + if len(cluster) == 1 { + continue + } + fmt.Printf("[%d] cluster created:\n", i) + for _, request := range cluster { + totalClusterCount++ + fmt.Printf("\t%v\n", request.ID) + } + totalRequestsSentNew++ + } + fmt.Printf("Reduced %d requests to %d via clustering\n", totalClusterCount, totalRequestsSentNew) +} + +type mockOutput struct{} + +// Close closes the output writer interface +func (m *mockOutput) Close() {} + +// Colorizer returns the colorizer instance for writer +func (m *mockOutput) Colorizer() aurora.Aurora { + return nil +} + +// Write writes the event to file and/or screen. +func (m *mockOutput) Write(*output.ResultEvent) error { + return nil +} + +// Request writes a log the requests trace log +func (m *mockOutput) Request(templateID, url, requestType string, err error) {} diff --git a/v2/pkg/protocols/common/compare/compare.go b/v2/pkg/protocols/common/compare/compare.go new file mode 100644 index 000000000..683b5facb --- /dev/null +++ b/v2/pkg/protocols/common/compare/compare.go @@ -0,0 +1,37 @@ +package compare + +import "strings" + +// StringSlice compares two string slices for equality +func StringSlice(a, b []string) bool { + // If one is nil, the other must also be nil. + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for i := range a { + if !strings.EqualFold(a[i], b[i]) { + return false + } + } + return true +} + +// StringMap compares two string maps for equality +func StringMap(a, b map[string]string) bool { + // If one is nil, the other must also be nil. + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || !strings.EqualFold(v, w) { + return false + } + } + return true +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 12f8a3653..a88c077f7 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -18,6 +18,11 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption return &Executer{requests: requests, options: options} } +// GetRequests returns the requests the rule will perform +func (e *Executer) GetRequests() []protocols.Request { + return e.requests +} + // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { for _, request := range e.requests { diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go new file mode 100644 index 000000000..68107c9eb --- /dev/null +++ b/v2/pkg/protocols/http/cluster.go @@ -0,0 +1,29 @@ +package http + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/compare" +) + +// CanCluster returns true if the request can be clustered. +// +// This used by the clustering engine to decide whether two requests +// are similar enough to be considered one and can be checked by +// just adding the matcher/extractors for the request and the correct IDs. +func (r *Request) CanCluster(other *Request) bool { + if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe { + return false + } + if r.Method != other.Method || + r.MaxRedirects != other.MaxRedirects || + r.CookieReuse != other.CookieReuse || + r.Redirects != other.Redirects { + return false + } + if !compare.StringSlice(r.Path, other.Path) { + return false + } + if !compare.StringMap(r.Headers, other.Headers) { + return false + } + return true +} From 9d6ab2754cd618f9c7158aef0f76c36ecde793be Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 13 Jan 2021 12:18:56 +0530 Subject: [PATCH 78/92] Added clustered requests executer to nuclei + misc --- v2/pkg/protocols/common/clusterer/executer.go | 100 ++++++++++++++++++ v2/pkg/protocols/common/executer/executer.go | 5 - .../protocols/common/generators/validate.go | 4 +- v2/pkg/protocols/dns/operators.go | 10 +- v2/pkg/protocols/dns/request.go | 7 +- v2/pkg/protocols/file/operators.go | 10 +- v2/pkg/protocols/file/request.go | 9 +- v2/pkg/protocols/http/operators.go | 10 +- v2/pkg/protocols/http/request.go | 11 +- v2/pkg/protocols/network/operators.go | 10 +- v2/pkg/protocols/network/request.go | 7 +- 11 files changed, 141 insertions(+), 42 deletions(-) create mode 100644 v2/pkg/protocols/common/clusterer/executer.go diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go new file mode 100644 index 000000000..4636613f5 --- /dev/null +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -0,0 +1,100 @@ +package clusterer + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" +) + +// Executer executes a group of requests for a protocol for a clustered +// request. It is different from normal executers since the original +// operators are all combined and post processed after making the request. +// +// TODO: We only cluster http requests as of now. +type Executer struct { + requests *http.Request + operators []*clusteredOperator + options *protocols.ExecuterOptions +} + +type clusteredOperator struct { + templateID string + templateInfo map[string]string + operator *operators.Operators +} + +var _ protocols.Executer = &Executer{} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOptions) *Executer { + executer := &Executer{ + options: options, + requests: requests[0].RequestsHTTP[0], + } + for _, req := range requests { + executer.operators = append(executer.operators, &clusteredOperator{ + templateID: req.ID, + templateInfo: req.Info, + operator: req.RequestsHTTP[0].CompiledOperators, + }) + } + return executer +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + return e.requests.Compile(e.options) +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int { + var count int + count += e.requests.Requests() + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + dynamicValues := make(map[string]interface{}) + err := e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + for _, operator := range e.operators { + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + if matched && result != nil { + event.OperatorsResult = result + event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-info"] = operator.templateInfo + event.Results = e.requests.MakeResultEvent(event) + results = true + for _, r := range event.Results { + e.options.Output.Write(r) + } + } + } + }) + if err != nil { + return results, err + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { + dynamicValues := make(map[string]interface{}) + _ = e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) { + for _, operator := range e.operators { + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + if matched && result != nil { + event.OperatorsResult = result + event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-info"] = operator.templateInfo + event.Results = e.requests.MakeResultEvent(event) + callback(event) + } + } + }) + return nil +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index a88c077f7..12f8a3653 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -18,11 +18,6 @@ func NewExecuter(requests []protocols.Request, options *protocols.ExecuterOption return &Executer{requests: requests, options: options} } -// GetRequests returns the requests the rule will perform -func (e *Executer) GetRequests() []protocols.Request { - return e.requests -} - // Compile compiles the execution generators preparing any requests possible. func (e *Executer) Compile() error { for _, request := range e.requests { diff --git a/v2/pkg/protocols/common/generators/validate.go b/v2/pkg/protocols/common/generators/validate.go index acc8a2da0..b6f3776b6 100644 --- a/v2/pkg/protocols/common/generators/validate.go +++ b/v2/pkg/protocols/common/generators/validate.go @@ -7,7 +7,7 @@ import ( "path" "strings" - "github.com/spf13/cast" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // validate validates the payloads if any. @@ -40,7 +40,7 @@ func (g *Generator) validate(payloads map[string]interface{}, templatePath strin return fmt.Errorf("the %s file for payload %s does not exist or does not contain enough elements", pt, name) } case interface{}: - loadedPayloads := cast.ToStringSlice(pt) + loadedPayloads := types.ToStringSlice(pt) if len(loadedPayloads) == 0 { return fmt.Errorf("the payload %s does not contain enough elements", name) } diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index d11476be7..d48076849 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -113,11 +113,13 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out rawData := resp.String() data["raw"] = rawData + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo return data } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) // If we have multiple matchers with names, write each of them separately. @@ -143,8 +145,8 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: r.options.TemplateID, - Info: r.options.TemplateInfo, + TemplateID: wrapped.InternalEvent["template-id"].(string), + Info: wrapped.InternalEvent["template-info"].(map[string]string), Type: "dns", Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 974d55ab8..1b299adea 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -57,11 +57,10 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) - if !ok { - return nil + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) } - event.OperatorsResult = result - event.Results = r.makeResultEvent(event) } callback(event) return nil diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index a8262360a..c8559e8ad 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -72,11 +72,13 @@ func (r *Request) responseToDSLMap(raw string, host, matched string) output.Inte data["host"] = host data["matched"] = matched data["raw"] = raw + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo return data } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) // If we have multiple matchers with names, write each of them separately. @@ -102,8 +104,8 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: r.options.TemplateID, - Info: r.options.TemplateInfo, + TemplateID: wrapped.InternalEvent["template-id"].(string), + Info: wrapped.InternalEvent["template-info"].(map[string]string), Type: "file", Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index b3461b976..29e56a940 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -51,13 +51,12 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) - if !ok { - return + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) } - event.OperatorsResult = result - event.Results = r.makeResultEvent(event) - callback(event) } + callback(event) }) if err != nil { r.options.Output.Request(r.options.TemplateID, input, "file", err) diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index f9462fd42..646ad1b1a 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -109,11 +109,13 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["raw"] = rawString } data["duration"] = duration.Seconds() + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo return data } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) // If we have multiple matchers with names, write each of them separately. @@ -139,8 +141,8 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: r.options.TemplateID, - Info: r.options.TemplateInfo, + TemplateID: wrapped.InternalEvent["template-id"].(string), + Info: wrapped.InternalEvent["template-info"].(map[string]string), Type: "http", Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index e1118792d..f49a932f8 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -331,14 +331,13 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) - if !ok { - return nil + if ok && result != nil { + event.OperatorsResult = result + result.PayloadValues = request.meta + event.Results = r.MakeResultEvent(event) } - event.OperatorsResult = result - result.PayloadValues = request.meta - event.Results = r.makeResultEvent(event) - callback(event) } + callback(event) return nil } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index c2748f739..2e9f3d85e 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -75,11 +75,13 @@ func (r *Request) responseToDSLMap(req, resp string, host, matched string) outpu data["request"] = req } data["data"] = resp + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo return data } -// makeResultEvent creates a result event from internal wrapped event -func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) // If we have multiple matchers with names, write each of them separately. @@ -105,8 +107,8 @@ func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ - TemplateID: r.options.TemplateID, - Info: r.options.TemplateInfo, + TemplateID: wrapped.InternalEvent["template-id"].(string), + Info: wrapped.InternalEvent["template-info"].(map[string]string), Type: "network", Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 23c676d2c..a34021632 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -123,11 +123,10 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) - if !ok { - return nil + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) } - event.OperatorsResult = result - event.Results = r.makeResultEvent(event) } callback(event) return nil From 3899542f69ef5fbd200b3f6bfca591c1ac68b5bd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 13 Jan 2021 12:58:23 +0530 Subject: [PATCH 79/92] Finished initial request clustering functionality --- v2/internal/runner/runner.go | 35 +++++++++++++++++-- v2/internal/runner/templates.go | 7 ++-- .../common/clusterer/clusterer_test.go | 3 +- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 738f5a335..2cfbafa5e 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -2,6 +2,7 @@ package runner import ( "bufio" + "fmt" "os" "strings" @@ -14,10 +15,13 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalogue" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" + "github.com/rs/xid" "go.uber.org/atomic" "go.uber.org/ratelimit" ) @@ -204,9 +208,34 @@ func (r *Runner) RunEnumeration() { } } + executerOpts := &protocols.ExecuterOptions{ + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalogue: r.catalogue, + RateLimiter: r.ratelimiter, + ProjectFile: r.projectFile, + } // pre-parse all the templates, apply filters + finalTemplates := []*templates.Template{} availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity) - templateCount := len(availableTemplates) + clusters := clusterer.Cluster(availableTemplates) + for _, cluster := range clusters { + if len(cluster) > 1 { + clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) + + gologger.Verbose().Msgf("Clustered %d requests together: %s", len(cluster), clusterID) + finalTemplates = append(finalTemplates, &templates.Template{ + ID: clusterID, + RequestsHTTP: cluster[0].RequestsHTTP, + Executer: clusterer.NewExecuter(cluster, executerOpts), + TotalRequests: 1, + }) + } else { + finalTemplates = append(finalTemplates, cluster[0]) + } + } + templateCount := len(finalTemplates) hasWorkflows := workflowCount > 0 // 0 matches means no templates were found in directory @@ -222,7 +251,7 @@ func (r *Runner) RunEnumeration() { // precompute total request count var totalRequests int64 = 0 - for _, t := range availableTemplates { + for _, t := range finalTemplates { // workflows will dynamically adjust the totals while running, as // it can't be know in advance which requests will be called if len(t.Workflows) > 0 { @@ -243,7 +272,7 @@ func (r *Runner) RunEnumeration() { p := r.progress p.Init(r.inputCount, templateCount, totalRequests) - for _, t := range availableTemplates { + for _, t := range finalTemplates { wgtemplates.Add() go func(template *templates.Template) { defer wgtemplates.Done() diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 50e39cf32..91fc0f583 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -13,12 +13,13 @@ import ( // getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered // by severity, along with a flag indicating if workflows are present. -func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []string) (parsedTemplates []*templates.Template, workflowCount int) { - workflowCount = 0 +func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []string) (map[string]*templates.Template, int) { + workflowCount := 0 filterBySeverity := len(severities) > 0 gologger.Info().Msgf("Loading templates...") + parsedTemplates := make(map[string]*templates.Template) for _, match := range templatePaths { t, err := r.parseTemplateFile(match) if err != nil { @@ -30,7 +31,7 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri } sev := strings.ToLower(t.Info["severity"]) if !filterBySeverity || hasMatchingSeverity(sev, severities) { - parsedTemplates = append(parsedTemplates, t) + parsedTemplates[t.ID] = t gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } else { gologger.Error().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities) diff --git a/v2/pkg/protocols/common/clusterer/clusterer_test.go b/v2/pkg/protocols/common/clusterer/clusterer_test.go index 1622b12fd..595a908c7 100644 --- a/v2/pkg/protocols/common/clusterer/clusterer_test.go +++ b/v2/pkg/protocols/common/clusterer/clusterer_test.go @@ -2,6 +2,7 @@ package clusterer import ( "fmt" + "log" "testing" "github.com/logrusorgru/aurora" @@ -37,7 +38,7 @@ func TestHTTPRequestsCluster(t *testing.T) { if _, ok := list[t.ID]; !ok { list[t.ID] = t } else { - // log.Fatalf("Duplicate template found: %v\n", t) + log.Printf("Duplicate template found: %v\n", t) } } From 04c349894ed691462f73084275befcc173335cdb Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 13 Jan 2021 13:03:07 +0530 Subject: [PATCH 80/92] Fixed threads and added race count to request total --- v2/pkg/protocols/http/http.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 3953cb75b..ed85fd47e 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -132,21 +132,14 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { func (r *Request) Requests() int { if r.generator != nil { payloadRequests := r.generator.NewIterator().Total() * len(r.Raw) - if r.Threads != 0 { - payloadRequests = payloadRequests * r.Threads - } return payloadRequests } if len(r.Raw) > 0 { requests := len(r.Raw) - if r.Threads != 0 { - requests = requests * r.Threads + if requests == 1 && r.RaceNumberRequests != 0 { + requests = requests * r.RaceNumberRequests } return requests } - requests := len(r.Path) - if r.Threads != 0 { - requests = requests * r.Threads - } - return requests + return len(r.Path) } From fa5a3d472924b4eab4bcc121f29bcacef9a27a27 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Jan 2021 12:22:19 +0530 Subject: [PATCH 81/92] Better ignore functionality with ignore tests --- v2/pkg/catalogue/ignore.go | 28 ++++++++++------------- v2/pkg/catalogue/ignore_test.go | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 v2/pkg/catalogue/ignore_test.go diff --git a/v2/pkg/catalogue/ignore.go b/v2/pkg/catalogue/ignore.go index 591d03b44..21749357f 100644 --- a/v2/pkg/catalogue/ignore.go +++ b/v2/pkg/catalogue/ignore.go @@ -37,15 +37,6 @@ func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { } for _, paths := range c.ignoreFiles { - // If we have a path to ignore, check if it's in the item. - if paths[len(paths)-1] == '/' { - if strings.Contains(item, paths) { - return true - } - - continue - } - // Check for file based extension in ignores if strings.HasSuffix(item, paths) { return true } @@ -55,15 +46,18 @@ func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { // ignoreFilesWithExcludes ignores results with exclude paths func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string { - excludeMap := make(map[string]struct{}, len(excluded)) - for _, excl := range excluded { - excludeMap[excl] = struct{}{} - } + var templates []string - templates := make([]string, 0, len(results)) - for _, incl := range results { - if _, found := excludeMap[incl]; !found { - templates = append(templates, incl) + for _, result := range results { + matched := false + for _, paths := range excluded { + if strings.HasSuffix(result, paths) { + matched = true + break + } + } + if !matched { + templates = append(templates, result) } } return templates diff --git a/v2/pkg/catalogue/ignore_test.go b/v2/pkg/catalogue/ignore_test.go new file mode 100644 index 000000000..136a95c7c --- /dev/null +++ b/v2/pkg/catalogue/ignore_test.go @@ -0,0 +1,39 @@ +package catalogue + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIgnoreFilesIgnore(t *testing.T) { + c := &Catalogue{ + ignoreFiles: []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}, + templatesDirectory: "test", + } + tests := []struct { + path string + ignore bool + }{ + {"workflows/", true}, + {"misc", false}, + {"cves/", false}, + {"cves/2020/cve-2020-5432.yaml", true}, + {"/Users/test/nuclei-templates/workflows/", true}, + {"/Users/test/nuclei-templates/misc", false}, + {"/Users/test/nuclei-templates/cves/", false}, + {"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true}, + } + for _, test := range tests { + require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), "could not ignore file correctly") + } +} + +func TestExcludeFilesIgnore(t *testing.T) { + c := &Catalogue{} + excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"} + paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"} + + data := c.ignoreFilesWithExcludes(paths, excludes) + require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files") +} From 9d0bb3a583791c8f1842f1fdc21628cc4b21518c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Jan 2021 13:21:21 +0530 Subject: [PATCH 82/92] Misc stuff, added timestamp to output + logging --- v2/internal/runner/options.go | 9 +++++++-- v2/internal/runner/runner.go | 8 +------- v2/internal/runner/templates.go | 3 ++- v2/pkg/catalogue/ignore.go | 5 +++++ v2/pkg/output/format_json.go | 7 ++++++- v2/pkg/output/output.go | 3 +++ 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index bf50a5ac5..88db27204 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -8,11 +8,17 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // ParseOptions parses the command line flags provided by a user func ParseOptions(options *types.Options) { + err := protocolinit.Init(options) + if err != nil { + gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) + } + // Check if stdin pipe was given options.Stdin = hasStdin() @@ -37,8 +43,7 @@ func ParseOptions(options *types.Options) { // Validate the options passed by the user and if any // invalid options have been used, exit. - err := validateOptions(options) - if err != nil { + if err = validateOptions(options); err != nil { gologger.Fatal().Msgf("Program exiting: %s\n", err) } } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2cfbafa5e..57a7a2f38 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -17,7 +17,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -180,11 +179,6 @@ func (r *Runner) Close() { // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { - err := protocolinit.Init(r.options) - if err != nil { - gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) - } - // resolves input templates definitions and any optional exclusion includedTemplates := r.catalogue.GetTemplatesPath(r.options.Templates) excludedTemplates := r.catalogue.GetTemplatesPath(r.options.ExcludedTemplates) @@ -229,7 +223,7 @@ func (r *Runner) RunEnumeration() { ID: clusterID, RequestsHTTP: cluster[0].RequestsHTTP, Executer: clusterer.NewExecuter(cluster, executerOpts), - TotalRequests: 1, + TotalRequests: len(cluster[0].RequestsHTTP), }) } else { finalTemplates = append(finalTemplates, cluster[0]) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 91fc0f583..15620d11d 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -73,8 +73,9 @@ func (r *Runner) logAvailableTemplate(tplPath string) { t, err := r.parseTemplateFile(tplPath) if err != nil { gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err) + } else { + gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } - gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, t.Info["name"], t.Info["author"], t.Info["severity"])) } // ListAvailableTemplates prints available templates to stdout diff --git a/v2/pkg/catalogue/ignore.go b/v2/pkg/catalogue/ignore.go index 21749357f..95351cfba 100644 --- a/v2/pkg/catalogue/ignore.go +++ b/v2/pkg/catalogue/ignore.go @@ -5,6 +5,8 @@ import ( "os" "path" "strings" + + "github.com/projectdiscovery/gologger" ) const nucleiIgnoreFile = ".nuclei-ignore" @@ -38,6 +40,7 @@ func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { for _, paths := range c.ignoreFiles { if strings.HasSuffix(item, paths) { + gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item) return true } } @@ -58,6 +61,8 @@ func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string } if !matched { templates = append(templates, result) + } else { + gologger.Error().Msgf("Excluding %s due to excludes filter", result) } } return templates diff --git a/v2/pkg/output/format_json.go b/v2/pkg/output/format_json.go index 0ca22961e..385bb6166 100644 --- a/v2/pkg/output/format_json.go +++ b/v2/pkg/output/format_json.go @@ -1,8 +1,13 @@ package output -import jsoniter "github.com/json-iterator/go" +import ( + "time" + + jsoniter "github.com/json-iterator/go" +) // formatJSON formats the output for json based formatting func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) { + output.Timestamp = time.Now() return jsoniter.Marshal(output) } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 4c5e5db4f..aeeb92ddc 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -4,6 +4,7 @@ import ( "os" "regexp" "sync" + "time" jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" @@ -72,6 +73,8 @@ type ResultEvent struct { Response string `json:"response,omitempty"` // Metadata contains any optional metadata for the event Metadata map[string]interface{} `json:"meta,omitempty"` + // Timestamp is the time the result was found at. + Timestamp time.Time `json:"timestamp"` } // NewStandardWriter creates a new output writer based on user configurations From 7403ade43775fe7d6b3576cb5843fc884de81f90 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Jan 2021 13:24:50 +0530 Subject: [PATCH 83/92] Fixed panic with list templates --- v2/internal/runner/runner.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 57a7a2f38..c4537b782 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -48,6 +48,11 @@ func New(options *types.Options) (*Runner, error) { if err := runner.updateTemplates(); err != nil { gologger.Warning().Msgf("Could not update templates: %s\n", err) } + // Read nucleiignore file if given a templateconfig + if runner.templatesConfig != nil { + runner.readNucleiIgnoreFile() + } + runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) // output coloring useColor := !options.NoColor @@ -62,12 +67,6 @@ func New(options *types.Options) (*Runner, error) { if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates { os.Exit(0) } - // Read nucleiignore file if given a templateconfig - if runner.templatesConfig != nil { - runner.readNucleiIgnoreFile() - } - runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) - if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil { gologger.Fatal().Msgf("Could not create temporary input file: %s\n", err) } else { From 8b93d5e1d2db1acc175adf80cb37d39690189262 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Jan 2021 18:27:48 +0530 Subject: [PATCH 84/92] Fixed extractions + input path clear + misc --- v2/pkg/operators/operators.go | 2 +- v2/pkg/output/output.go | 2 +- v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/file/request.go | 2 +- v2/pkg/protocols/http/build_request.go | 5 ++--- v2/pkg/protocols/http/raw/raw.go | 3 --- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/network/request.go | 2 +- 8 files changed, 8 insertions(+), 12 deletions(-) diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 818cba737..63581f0a8 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -121,7 +121,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. - if len(result.Extracts) > 0 || matches { + if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches { return result, true } return nil, false diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index aeeb92ddc..e79c1f4a5 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -54,7 +54,7 @@ type ResultEvent struct { // TemplateID is the ID of the template for the result. TemplateID string `json:"templateID"` // Info contains information block of the template for the result. - Info map[string]string `json:"info"` + Info map[string]string `json:"info,inline"` // MatcherName is the name of the matcher matched if any. MatcherName string `json:"matcher_name,omitempty"` // ExtractorName is the name of the extractor matched if any. diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 1b299adea..7b95cad43 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -56,7 +56,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { - result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 29e56a940..dd054c6ab 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -50,7 +50,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { - result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index b2c515bad..958dd0778 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -105,6 +105,8 @@ type generatedRequest struct { // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}) (*generatedRequest, error) { + baseURL = strings.TrimSuffix(baseURL, "/") + data, payloads, ok := r.nextValue() if !ok { return nil, io.EOF @@ -155,9 +157,6 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string { // MakeHTTPRequestFromModel creates a *http.Request from a request template func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*generatedRequest, error) { - if strings.HasSuffix(values["BaseURL"].(string), "/") { - data = strings.TrimPrefix(data, "/") - } URL := replacer.New(values).Replace(data) // Build a request on the specified URL diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 000c21eef..27fb57680 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -96,9 +96,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } else if strings.HasPrefix(rawRequest.Path, "?") { rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) } - if strings.HasSuffix(hostURL, "/") { - rawRequest.Path = strings.TrimPrefix(rawRequest.Path, "/") - } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) // Set the request body diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index f49a932f8..b08685453 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -330,7 +330,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { - result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result result.PayloadValues = request.meta diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index a34021632..408caa310 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -122,7 +122,7 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} if r.CompiledOperators != nil { - result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) From 6a739c2d0cb0ba712e245db9f1e4295d35ac0bb9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 14 Jan 2021 22:43:08 +0530 Subject: [PATCH 85/92] Limit concurrency in file protocol --- v2/pkg/protocols/file/request.go | 84 ++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index dd054c6ab..f9f3e537f 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -10,54 +10,64 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(r.options.Options.RateLimit) + err := r.getInputPaths(input, func(data string) { - file, err := os.Open(data) - if err != nil { - gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) - return - } - defer file.Close() + wg.Add() - stat, err := file.Stat() - if err != nil { - gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) - return - } - if stat.Size() >= int64(r.MaxSize) { - gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) - return - } + go func(data string) { + defer wg.Done() - buffer, err := ioutil.ReadAll(file) - if err != nil { - gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) - return - } - dataStr := tostring.UnsafeToString(buffer) - - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) - fmt.Fprintf(os.Stderr, "%s\n", dataStr) - } - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - ouputEvent := r.responseToDSLMap(dataStr, input, data) - - event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) + file, err := os.Open(data) + if err != nil { + gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + return } - } - callback(event) + defer file.Close() + + stat, err := file.Stat() + if err != nil { + gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + return + } + if stat.Size() >= int64(r.MaxSize) { + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + return + } + + buffer, err := ioutil.ReadAll(file) + if err != nil { + gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + return + } + dataStr := tostring.UnsafeToString(buffer) + + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) + fmt.Fprintf(os.Stderr, "%s\n", dataStr) + } + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) + ouputEvent := r.responseToDSLMap(dataStr, input, data) + + event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} + if r.CompiledOperators != nil { + result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) + } + } + callback(event) + }(data) }) + wg.Wait() if err != nil { r.options.Output.Request(r.options.TemplateID, input, "file", err) r.options.Progress.DecrementRequests(1) From ba7184ba58d587efa70d7dca4f03b91ae1be9736 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 15 Jan 2021 14:17:34 +0530 Subject: [PATCH 86/92] Added back cookie-reuse functionality --- v2/internal/runner/runner.go | 38 ++++++++++++------- v2/pkg/protocols/http/http.go | 1 + .../http/httpclientpool/clientpool.go | 27 +++++++++++-- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c4537b782..544d015db 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -212,12 +212,23 @@ func (r *Runner) RunEnumeration() { // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity) + + var unclusteredRequests int64 = 0 + for _, template := range availableTemplates { + // workflows will dynamically adjust the totals while running, as + // it can't be know in advance which requests will be called + if len(template.Workflows) > 0 { + continue + } + unclusteredRequests += int64(template.TotalRequests) * r.inputCount + } + + originalTemplatesCount := len(availableTemplates) clusters := clusterer.Cluster(availableTemplates) for _, cluster := range clusters { if len(cluster) > 1 { clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) - gologger.Verbose().Msgf("Clustered %d requests together: %s", len(cluster), clusterID) finalTemplates = append(finalTemplates, &templates.Template{ ID: clusterID, RequestsHTTP: cluster[0].RequestsHTTP, @@ -228,7 +239,18 @@ func (r *Runner) RunEnumeration() { finalTemplates = append(finalTemplates, cluster[0]) } } - templateCount := len(finalTemplates) + + var totalRequests int64 = 0 + for _, t := range finalTemplates { + if len(t.Workflows) > 0 { + continue + } + totalRequests += int64(t.TotalRequests) * r.inputCount + } + if totalRequests < unclusteredRequests { + gologger.Info().Msgf("Reduced %d requests to %d via clustering", unclusteredRequests, totalRequests) + } + templateCount := originalTemplatesCount hasWorkflows := workflowCount > 0 // 0 matches means no templates were found in directory @@ -241,18 +263,6 @@ func (r *Runner) RunEnumeration() { r.colorizer.Bold(templateCount-workflowCount).String(), r.colorizer.Bold(workflowCount).String()) - // precompute total request count - var totalRequests int64 = 0 - - for _, t := range finalTemplates { - // workflows will dynamically adjust the totals while running, as - // it can't be know in advance which requests will be called - if len(t.Workflows) > 0 { - continue - } - totalRequests += int64(t.TotalRequests) * r.inputCount - } - results := &atomic.Bool{} wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) // Starts polling or ignore diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index ed85fd47e..e186c545c 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -77,6 +77,7 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { Threads: r.Threads, MaxRedirects: r.MaxRedirects, FollowRedirects: r.Redirects, + CookieReuse: r.CookieReuse, }) if err != nil { return errors.Wrap(err, "could not get dns client") diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 4ae10f87a..6e41741c6 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "net/http/cookiejar" "net/url" "strconv" "strings" @@ -18,6 +19,7 @@ import ( "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" "golang.org/x/net/proxy" + "golang.org/x/net/publicsuffix" ) var ( @@ -47,6 +49,8 @@ func Init(options *types.Options) error { // Configuration contains the custom configuration options for a client type Configuration struct { + // CookieReuse enables cookie reuse for the http client (cookiejar impl) + CookieReuse bool // Threads contains the threads for the client Threads int // MaxRedirects is the maximum number of redirects to follow @@ -65,6 +69,8 @@ func (c *Configuration) Hash() string { builder.WriteString(strconv.Itoa(c.MaxRedirects)) builder.WriteString("f") builder.WriteString(strconv.FormatBool(c.FollowRedirects)) + builder.WriteString("r") + builder.WriteString(strconv.FormatBool(c.CookieReuse)) hash := builder.String() return hash } @@ -79,7 +85,7 @@ func GetRawHTTP() *rawhttp.Client { // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryablehttp.Client, error) { - if !(configuration.Threads > 0 && configuration.MaxRedirects > 0 && configuration.FollowRedirects) { + if configuration.Threads == 0 && configuration.MaxRedirects == 0 && !configuration.FollowRedirects && !configuration.CookieReuse { return normalClient, nil } return wrappedGet(options, configuration) @@ -166,16 +172,29 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl transport.Proxy = http.ProxyURL(proxyURL) } + var jar *cookiejar.Jar + if configuration.CookieReuse { + if jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}); err != nil { + return nil, errors.Wrap(err, "could not create cookiejar") + } + } + client := retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), }, retryablehttpOptions) + if jar != nil { + client.HTTPClient.Jar = jar + } client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - poolMutex.Lock() - clientPool[hash] = client - poolMutex.Unlock() + // Only add to client pool if we don't have a cookie jar in place. + if jar == nil { + poolMutex.Lock() + clientPool[hash] = client + poolMutex.Unlock() + } return client, nil } From 517da74dea9ff0c4b8ad9e92c629cdf5d60be167 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 15 Jan 2021 14:22:05 +0530 Subject: [PATCH 87/92] Fixed nuclei custom headers not working --- v2/pkg/protocols/http/http.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index e186c545c..bebd26b3b 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -84,6 +84,9 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } r.httpClient = client r.options = options + for _, option := range r.options.Options.CustomHeaders { + r.customHeaders = append(r.customHeaders, option) + } if len(r.Raw) > 0 { r.rawhttpClient = httpclientpool.GetRawHTTP() From 621f4b2c0464b6231f8ec6e3cf7976963cad68bd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 15 Jan 2021 20:35:15 +0530 Subject: [PATCH 88/92] Added more info to update --- v2/internal/runner/update.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 04fceba08..162b09c31 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -257,6 +257,8 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string return fmt.Errorf("could not write template file: %s", err) } f.Close() + + gologger.Info().Msgf("Download new template: %s\n", templatePath) checksums[templatePath] = hex.EncodeToString(hasher.Sum(nil)) } @@ -267,6 +269,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string _, ok := checksums[k] if !ok && v[0] == v[1] { os.Remove(k) + gologger.Info().Msgf("Removing stale template: %s\n", k) } } } From 9f60bbcdb7656ef99c05df250dc5b5e9f63bede1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 15 Jan 2021 20:49:44 +0530 Subject: [PATCH 89/92] Fixed ignore logic + update printing --- v2/internal/runner/update.go | 6 +++++- v2/pkg/catalogue/ignore.go | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 162b09c31..f060888a8 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -220,6 +220,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") previousChecksum := readPreviousTemplatesChecksum(checksumFile) + addedTemplates, deletedTemplates := 0, 0 checksums := make(map[string]string) for _, file := range z.File { directory, name := filepath.Split(file.Name) @@ -258,6 +259,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string } f.Close() + addedTemplates++ gologger.Info().Msgf("Download new template: %s\n", templatePath) checksums[templatePath] = hex.EncodeToString(hasher.Sum(nil)) } @@ -268,11 +270,13 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string for k, v := range previousChecksum { _, ok := checksums[k] if !ok && v[0] == v[1] { + deletedTemplates++ os.Remove(k) - gologger.Info().Msgf("Removing stale template: %s\n", k) + gologger.Error().Lable("INF").Msgf("Removing stale template: %s\n", k) } } } + gologger.Info().Msgf("Added %d templates, removed %d templates", addedTemplates, deletedTemplates) return writeTemplatesChecksum(checksumFile, checksums) } diff --git a/v2/pkg/catalogue/ignore.go b/v2/pkg/catalogue/ignore.go index 95351cfba..4110f3c82 100644 --- a/v2/pkg/catalogue/ignore.go +++ b/v2/pkg/catalogue/ignore.go @@ -39,7 +39,13 @@ func (c *Catalogue) checkIfInNucleiIgnore(item string) bool { } for _, paths := range c.ignoreFiles { - if strings.HasSuffix(item, paths) { + dir := path.Dir(item) + + if strings.EqualFold(dir, paths) { + gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item) + return true + } + if strings.HasSuffix(paths, ".yaml") && strings.HasSuffix(item, paths) { gologger.Error().Msgf("Excluding %s due to nuclei-ignore filter", item) return true } @@ -54,7 +60,13 @@ func (c *Catalogue) ignoreFilesWithExcludes(results, excluded []string) []string for _, result := range results { matched := false for _, paths := range excluded { - if strings.HasSuffix(result, paths) { + dir := path.Dir(result) + + if strings.EqualFold(dir, paths) { + matched = true + break + } + if strings.HasSuffix(paths, ".yaml") && strings.HasSuffix(result, paths) { matched = true break } From d9ba877944e4b42b3921f2bf29c361f1f7b26136 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 15 Jan 2021 22:05:10 +0530 Subject: [PATCH 90/92] Added update changelog + fixed update-templates bugs --- v2/internal/runner/update.go | 86 ++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index f060888a8..e8ec1c2cb 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -15,11 +15,13 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "time" "github.com/blang/semver" "github.com/google/go-github/v32/github" + "github.com/olekukonko/tablewriter" "github.com/projectdiscovery/gologger" ) @@ -57,10 +59,10 @@ func (r *Runner) updateTemplates() error { } // Use custom location if user has given a template directory - if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { - home = r.options.TemplatesDirectory - } r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")} + if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { + r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory + } // Download the repository and also write the revision to a HEAD file. version, asset, getErr := r.getLatestReleaseFromGithub() @@ -69,7 +71,7 @@ func (r *Runner) updateTemplates() error { } gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) - err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL()) + err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } @@ -121,13 +123,12 @@ func (r *Runner) updateTemplates() error { } if r.options.TemplatesDirectory != "" { - home = r.options.TemplatesDirectory - r.templatesConfig.TemplatesDirectory = path.Join(home, "nuclei-templates") + r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory } r.templatesConfig.CurrentVersion = version.String() - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", "update-templates", version.String(), r.templatesConfig.TemplatesDirectory) - err = r.downloadReleaseAndUnzip(ctx, asset.GetZipballURL()) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } @@ -180,7 +181,7 @@ func (r *Runner) getLatestReleaseFromGithub() (semver.Version, *github.Repositor } // downloadReleaseAndUnzip downloads and unzips the release in a directory -func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string) error { +func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadURL string) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { return fmt.Errorf("failed to create HTTP request to %s: %s", downloadURL, err) @@ -212,6 +213,8 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string return fmt.Errorf("failed to create template base folder: %s", err) } + totalCount := 0 + additions, deletions, modifications := []string{}, []string{}, []string{} // We use file-checksums that are md5 hashes to store the list of files->hashes // that have been downloaded previously. // If the path isn't found in new update after being read from the previous checksum, @@ -219,18 +222,19 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string // as well as solves a long problem with nuclei-template updates. checksumFile := path.Join(r.templatesConfig.TemplatesDirectory, ".checksum") previousChecksum := readPreviousTemplatesChecksum(checksumFile) - - addedTemplates, deletedTemplates := 0, 0 checksums := make(map[string]string) for _, file := range z.File { directory, name := filepath.Split(file.Name) if name == "" { continue } - paths := strings.Split(directory, "/") finalPath := strings.Join(paths[1:], "/") + if strings.HasPrefix(name, ".") || strings.HasPrefix(finalPath, ".") || strings.EqualFold(name, "README.md") { + continue + } + totalCount++ templateDirectory := path.Join(r.templatesConfig.TemplatesDirectory, finalPath) err = os.MkdirAll(templateDirectory, os.ModePerm) if err != nil { @@ -238,6 +242,11 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string } templatePath := path.Join(templateDirectory, name) + + isAddition := false + if _, err := os.Stat(templatePath); os.IsNotExist(err) { + isAddition = true + } f, err := os.OpenFile(templatePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777) if err != nil { f.Close() @@ -259,8 +268,11 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string } f.Close() - addedTemplates++ - gologger.Info().Msgf("Download new template: %s\n", templatePath) + if isAddition { + additions = append(additions, path.Join(finalPath, name)) + } else { + modifications = append(modifications, path.Join(finalPath, name)) + } checksums[templatePath] = hex.EncodeToString(hasher.Sum(nil)) } @@ -270,13 +282,12 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, downloadURL string for k, v := range previousChecksum { _, ok := checksums[k] if !ok && v[0] == v[1] { - deletedTemplates++ os.Remove(k) - gologger.Error().Lable("INF").Msgf("Removing stale template: %s\n", k) + deletions = append(deletions, strings.TrimPrefix(strings.TrimPrefix(k, r.templatesConfig.TemplatesDirectory), "/")) } } } - gologger.Info().Msgf("Added %d templates, removed %d templates", addedTemplates, deletedTemplates) + r.printUpdateChangelog(additions, modifications, deletions, version, totalCount) return writeTemplatesChecksum(checksumFile, checksums) } @@ -308,12 +319,14 @@ func readPreviousTemplatesChecksum(file string) map[string][2]string { if err != nil { continue } - defer f.Close() hasher := md5.New() if _, err := io.Copy(hasher, f); err != nil { + f.Close() continue } + f.Close() + values[1] = hex.EncodeToString(hasher.Sum(nil)) checksum[parts[0]] = values } @@ -324,7 +337,7 @@ func readPreviousTemplatesChecksum(file string) map[string][2]string { func writeTemplatesChecksum(file string, checksum map[string]string) error { f, err := os.Create(file) if err != nil { - return nil + return err } defer f.Close() @@ -336,3 +349,38 @@ func writeTemplatesChecksum(file string, checksum map[string]string) error { } return nil } + +func (r *Runner) printUpdateChangelog(additions, modifications, deletions []string, version string, totalCount int) { + if len(additions) > 0 { + gologger.Print().Msgf("\nNew additions: \n\n") + + for _, addition := range additions { + gologger.Print().Msgf("%s", addition) + } + } + if len(modifications) > 0 { + gologger.Print().Msgf("\nModifications: \n\n") + + for _, modification := range modifications { + gologger.Print().Msgf("%s", modification) + } + } + if len(deletions) > 0 { + gologger.Print().Msgf("\nDeletions: \n\n") + + for _, deletion := range deletions { + gologger.Print().Msgf("%s", deletion) + } + } + + gologger.Print().Msgf("\nNuclei Templates v%s Changelog\n", version) + data := [][]string{ + {strconv.Itoa(totalCount), strconv.Itoa(len(additions)), strconv.Itoa(len(modifications)), strconv.Itoa(len(deletions))}, + } + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Total", "New", "Modifications", "Deletions"}) + for _, v := range data { + table.Append(v) + } + table.Render() +} From 9d27f79cb159f1e5a72f3769820f166b486363c7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 16 Jan 2021 12:06:27 +0530 Subject: [PATCH 91/92] Exposed IP information in protocols results --- v2/pkg/output/output.go | 2 ++ .../protocols/http/httpclientpool/clientpool.go | 8 ++++---- v2/pkg/protocols/http/operators.go | 1 + v2/pkg/protocols/http/request.go | 16 +++++++++++++--- v2/pkg/protocols/network/operators.go | 1 + v2/pkg/protocols/network/request.go | 12 +++++++++--- 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index e79c1f4a5..002d404f4 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -73,6 +73,8 @@ type ResultEvent struct { Response string `json:"response,omitempty"` // Metadata contains any optional metadata for the event Metadata map[string]interface{} `json:"meta,omitempty"` + // IP is the IP address for the found result event. + IP string `json:"ip,omitempty"` // Timestamp is the time the result was found at. Timestamp time.Time `json:"timestamp"` } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index 6e41741c6..b2e8d273e 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -23,7 +23,7 @@ import ( ) var ( - dialer *fastdialer.Dialer + Dialer *fastdialer.Dialer rawhttpClient *rawhttp.Client poolMutex *sync.RWMutex normalClient *retryablehttp.Client @@ -96,8 +96,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl var proxyURL *url.URL var err error - if dialer == nil { - dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions) + if Dialer == nil { + Dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions) } if err != nil { return nil, errors.Wrap(err, "could not create dialer") @@ -139,7 +139,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl maxRedirects := configuration.MaxRedirects transport := &http.Transport{ - DialContext: dialer.Dial, + DialContext: Dialer.Dial, MaxIdleConns: maxIdleConns, MaxIdleConnsPerHost: maxIdleConnsPerHost, MaxConnsPerHost: maxConnsPerHost, diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 646ad1b1a..32f2ea919 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -148,6 +148,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Matched: wrapped.InternalEvent["matched"].(string), Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + IP: wrapped.InternalEvent["ip"].(string), } if r.options.Options.JSONRequests { data.Request = wrapped.InternalEvent["request"].(string) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index b08685453..7ec702be1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/rawhttp" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" @@ -235,12 +236,19 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam } var formedURL string + var hostname string timeStart := time.Now() if request.original.Pipeline { formedURL = request.rawRequest.FullURL + if parsed, err := url.Parse(formedURL); err == nil { + hostname = parsed.Hostname() + } resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) } else if request.original.Unsafe { formedURL = request.rawRequest.FullURL + if parsed, err := url.Parse(formedURL); err == nil { + hostname = parsed.Hostname() + } request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n") options := request.original.rawhttpClient.Options options.AutomaticContentLength = !r.DisableAutoContentLength @@ -248,6 +256,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam options.FollowRedirects = r.Redirects resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) } else { + hostname = request.request.URL.Hostname() formedURL = request.request.URL.String() // if nuclei-project is available check if the request was already sent previously if r.options.ProjectFile != nil { @@ -326,11 +335,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam if request.request != nil { matchedURL = request.request.URL.String() } - ouputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) - event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result result.PayloadValues = request.meta diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 2e9f3d85e..7007f6e3c 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -113,6 +113,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + IP: wrapped.InternalEvent["ip"].(string), } if r.options.Options.JSONRequests { data.Request = wrapped.InternalEvent["request"].(string) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 408caa310..968770c18 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -56,6 +56,11 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback return err } + var hostname string + if host, _, err := net.SplitHostPort(actualAddress); err == nil { + hostname = host + } + conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress) if err != nil { r.options.Output.Request(r.options.TemplateID, address, "network", err) @@ -118,11 +123,12 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) fmt.Fprintf(os.Stderr, "%s\n", resp) } - ouputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress) + outputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress) + outputEvent["ip"] = r.dialer.GetDialedIP(hostname) - event := &output.InternalWrappedEvent{InternalEvent: ouputEvent} + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract) + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) if ok && result != nil { event.OperatorsResult = result event.Results = r.MakeResultEvent(event) From a50bc4c30f43cc3447ce5804888cd0042162f19b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 16 Jan 2021 12:26:38 +0530 Subject: [PATCH 92/92] Added matched count + misc --- v2/internal/progress/progress.go | 13 ++++++++ v2/internal/runner/runner.go | 9 ++--- v2/pkg/protocols/common/clusterer/executer.go | 1 + v2/pkg/protocols/common/executer/executer.go | 1 + v2/pkg/workflows/execute_test.go | 33 +++++++++++++++---- 5 files changed, 47 insertions(+), 10 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 64cb7de01..cfa6b3151 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -67,6 +67,7 @@ func (p *Progress) Init(hostCount int64, rulesCount int, requestCount int64) { p.stats.AddStatic("startedAt", time.Now()) p.stats.AddCounter("requests", uint64(0)) p.stats.AddCounter("errors", uint64(0)) + p.stats.AddCounter("matched", uint64(0)) p.stats.AddCounter("total", uint64(requestCount)) if p.active { @@ -86,6 +87,11 @@ func (p *Progress) IncrementRequests() { p.stats.IncrementCounter("requests", 1) } +// IncrementMatched increments the matched counter by 1. +func (p *Progress) IncrementMatched() { + p.stats.IncrementCounter("matched", 1) +} + // DecrementRequests decrements the number of requests from total. func (p *Progress) DecrementRequests(count int64) { // mimic dropping by incrementing the completed requests @@ -119,6 +125,11 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { builder.WriteString(" | RPS: ") builder.WriteString(clistats.String(uint64(float64(requests) / duration.Seconds()))) + matched, _ := stats.GetCounter("matched") + + builder.WriteString(" | Matched: ") + builder.WriteString(clistats.String(matched)) + errors, _ := stats.GetCounter("errors") builder.WriteString(" | Errors: ") builder.WriteString(clistats.String(errors)) @@ -153,6 +164,8 @@ func (p *Progress) getMetrics() map[string]interface{} { results["templates"] = clistats.String(templates) hosts, _ := p.stats.GetStatic("hosts") results["hosts"] = clistats.String(hosts) + matched, _ := p.stats.GetStatic("matched") + results["matched"] = clistats.String(matched) requests, _ := p.stats.GetCounter("requests") results["requests"] = clistats.String(requests) total, _ := p.stats.GetCounter("total") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 544d015db..f0c845622 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -224,6 +224,7 @@ func (r *Runner) RunEnumeration() { } originalTemplatesCount := len(availableTemplates) + clusterCount := 0 clusters := clusterer.Cluster(availableTemplates) for _, cluster := range clusters { if len(cluster) > 1 { @@ -235,6 +236,7 @@ func (r *Runner) RunEnumeration() { Executer: clusterer.NewExecuter(cluster, executerOpts), TotalRequests: len(cluster[0].RequestsHTTP), }) + clusterCount++ } else { finalTemplates = append(finalTemplates, cluster[0]) } @@ -248,7 +250,7 @@ func (r *Runner) RunEnumeration() { totalRequests += int64(t.TotalRequests) * r.inputCount } if totalRequests < unclusteredRequests { - gologger.Info().Msgf("Reduced %d requests to %d via clustering", unclusteredRequests, totalRequests) + gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount) } templateCount := originalTemplatesCount hasWorkflows := workflowCount > 0 @@ -272,8 +274,7 @@ func (r *Runner) RunEnumeration() { gologger.Error().Msgf("Could not find any valid input URLs.") } else if totalRequests > 0 || hasWorkflows { // tracks global progress and captures stdout/stderr until p.Wait finishes - p := r.progress - p.Init(r.inputCount, templateCount, totalRequests) + r.progress.Init(r.inputCount, templateCount, totalRequests) for _, t := range finalTemplates { wgtemplates.Add() @@ -288,7 +289,7 @@ func (r *Runner) RunEnumeration() { }(t) } wgtemplates.Wait() - p.Stop() + r.progress.Stop() } if !results.Load() { diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 4636613f5..1a2a0508f 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -71,6 +71,7 @@ func (e *Executer) Execute(input string) (bool, error) { results = true for _, r := range event.Results { e.options.Output.Write(r) + e.options.Progress.IncrementMatched() } } } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 12f8a3653..a06bd0d71 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -51,6 +51,7 @@ func (e *Executer) Execute(input string) (bool, error) { for _, result := range event.Results { results = true e.options.Output.Write(result) + e.options.Progress.IncrementMatched() } }) if err != nil { diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 8bd345689..3a4b9d3e1 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -3,6 +3,7 @@ package workflows import ( "testing" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -10,9 +11,14 @@ import ( ) func TestWorkflowsSimple(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: true}}, - }} + }, + options: &protocols.ExecuterOptions{ + Progress: progress, + }} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -20,6 +26,8 @@ func TestWorkflowsSimple(t *testing.T) { } func TestWorkflowsSimpleMultiple(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: true, executeHook: func(input string) { @@ -28,7 +36,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { {Executer: &mockExecuter{result: true, executeHook: func(input string) { secondInput = input }}}, - }} + }, + options: &protocols.ExecuterOptions{Progress: progress}} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -39,6 +48,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { } func TestWorkflowsSubtemplates(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: true, executeHook: func(input string) { @@ -49,7 +60,8 @@ func TestWorkflowsSubtemplates(t *testing.T) { secondInput = input }}}, }}, - }} + }, + options: &protocols.ExecuterOptions{Progress: progress}} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -60,6 +72,8 @@ func TestWorkflowsSubtemplates(t *testing.T) { } func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: false, executeHook: func(input string) { @@ -70,7 +84,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { secondInput = input }}}, }}, - }} + }, + options: &protocols.ExecuterOptions{Progress: progress}} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -81,6 +96,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { } func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: true, executeHook: func(input string) { @@ -99,7 +116,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }}, }, }, - }} + }, + options: &protocols.ExecuterOptions{Progress: progress}} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow") @@ -110,6 +128,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { } func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { + progress, _ := progress.NewProgress(false, false, 0) + var firstInput, secondInput string workflow := &Workflow{Workflows: []*WorkflowTemplate{ {Executer: &mockExecuter{result: true, executeHook: func(input string) { @@ -128,7 +148,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }}, }, }, - }} + }, + options: &protocols.ExecuterOptions{Progress: progress}} matched, err := workflow.RunWorkflow("https://test.com") require.Nil(t, err, "could not run workflow")