From bbdfb565af33e34bd5ffb7680d885baffec10e5c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Dec 2020 14:54:20 +0530 Subject: [PATCH] Added network protocol support --- v2/internal/runner/runner.go | 4 + v2/pkg/protocols/network/executer.go | 90 +++++++++++++++ v2/pkg/protocols/network/network.go | 79 +++++++++++++ .../network/networkclientpool/clientpool.go | 38 ++++++ v2/pkg/protocols/network/operators.go | 108 ++++++++++++++++++ v2/pkg/protocols/network/request.go | 96 ++++++++++++++++ v2/pkg/templates/compile.go | 14 ++- v2/pkg/templates/templates.go | 3 + 8 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 v2/pkg/protocols/network/executer.go create mode 100644 v2/pkg/protocols/network/network.go create mode 100644 v2/pkg/protocols/network/networkclientpool/clientpool.go create mode 100644 v2/pkg/protocols/network/operators.go create mode 100644 v2/pkg/protocols/network/request.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f11fde039..9c40f3819 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -277,5 +278,8 @@ func (r *Runner) initializeProtocols() error { if err := httpclientpool.Init(r.options); err != nil { return err } + if err := networkclientpool.Init(r.options); err != nil { + return err + } return nil } diff --git a/v2/pkg/protocols/network/executer.go b/v2/pkg/protocols/network/executer.go new file mode 100644 index 000000000..52f59eb5e --- /dev/null +++ b/v2/pkg/protocols/network/executer.go @@ -0,0 +1,90 @@ +package network + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Executer executes a group of requests for a protocol +type Executer struct { + requests []*Request + options *protocols.ExecuterOptions +} + +var _ protocols.Executer = &Executer{} + +// NewExecuter creates a new request executer for list of requests +func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer { + return &Executer{requests: requests, options: options} +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *Executer) Compile() error { + for _, request := range e.requests { + err := request.Compile(e.options) + if err != nil { + return err + } + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (e *Executer) Requests() int { + var count int + for _, request := range e.requests { + count += int(request.Requests()) + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *Executer) Execute(input string) (bool, error) { + var results bool + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return false, err + } + if events == nil { + return false, nil + } + + // If we have a result field, we should add a result to slice. + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + + for _, result := range req.makeResultEvent(event) { + results = true + e.options.Output.Write(result) + } + } + } + return results, nil +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) { + var results []*output.InternalWrappedEvent + + for _, req := range e.requests { + events, err := req.ExecuteWithResults(input, nil) + if err != nil { + return nil, err + } + if events == nil { + return nil, nil + } + for _, event := range events { + if event.OperatorsResult == nil { + continue + } + event.Results = req.makeResultEvent(event) + } + results = append(results, events...) + } + return results, nil +} diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go new file mode 100644 index 000000000..234aed491 --- /dev/null +++ b/v2/pkg/protocols/network/network.go @@ -0,0 +1,79 @@ +package network + +import ( + "net" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" +) + +// Request contains a Network protocol request to be made from a template +type Request struct { + // Address is the address to send requests to (host:port combos generally) + Address string `yaml:"address"` + addressHost string + addressPort string + + // Payload is the payload to send for the network request + Payload string `yaml:"payload"` + // ReadSize is the size of response to read (1024 if not provided by default) + ReadSize int `yaml:"read-size"` + + // Operators for the current request go here. + operators.Operators `yaml:",inline"` + CompiledOperators *operators.Operators + + // cache any variables that may be needed for operation. + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + var err error + if strings.Contains(r.Address, ":") { + r.addressHost, r.addressPort, err = net.SplitHostPort(r.Address) + if err != nil { + return errors.Wrap(err, "could not parse address") + } + } else { + r.addressHost = r.Address + } + + // Create a client for the class + client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{}) + if err != nil { + return errors.Wrap(err, "could not get network client") + } + r.dialer = client + + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + r.options = options + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + return 1 +} + +// Make returns the request to be sent for the protocol +func (r *Request) Make(data string) (string, error) { + replacer := replacer.New(map[string]interface{}{"Address": data}) + address := replacer.Replace(r.addressHost) + if !strings.Contains(address, ":") { + address = net.JoinHostPort(address, r.addressPort) + } + return address, nil +} diff --git a/v2/pkg/protocols/network/networkclientpool/clientpool.go b/v2/pkg/protocols/network/networkclientpool/clientpool.go new file mode 100644 index 000000000..334bcea19 --- /dev/null +++ b/v2/pkg/protocols/network/networkclientpool/clientpool.go @@ -0,0 +1,38 @@ +package networkclientpool + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var ( + normalClient *fastdialer.Dialer +) + +// Init initializes the clientpool implementation +func Init(options *types.Options) error { + // Don't create clients if already created in past. + if normalClient != nil { + return nil + } + dialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions) + if err != nil { + return errors.Wrap(err, "could not create dialer") + } + normalClient = dialer + return nil +} + +// Configuration contains the custom configuration options for a client +type Configuration struct{} + +// Hash returns the hash of the configuration to allow client pooling +func (c *Configuration) Hash() string { + return "" +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (*fastdialer.Dialer, error) { + return normalClient, nil +} diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go new file mode 100644 index 000000000..8d85ee196 --- /dev/null +++ b/v2/pkg/protocols/network/operators.go @@ -0,0 +1,108 @@ +package network + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partString := matcher.Part + switch partString { + case "body", "all", "": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(itemStr))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(itemStr)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(itemStr)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(itemStr)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + part, ok := data[extractor.Part] + if !ok { + return nil + } + partString := part.(string) + + switch partString { + case "body", "all": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent { + data := make(output.InternalEvent, 4) + + // Some data regarding the request metadata + data["host"] = host + data["matched"] = matched + if r.options.Options.JSONRequests { + data["request"] = req + } + data["data"] = resp + return data +} + +// makeResultEvent creates a result event from internal wrapped event +func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + data := output.ResultEvent{ + TemplateID: r.options.TemplateID, + Info: r.options.TemplateInfo, + Type: "network", + Host: wrapped.InternalEvent["host"].(string), + Matched: wrapped.InternalEvent["matched"].(string), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + } + if r.options.Options.JSONRequests { + data.Request = wrapped.InternalEvent["request"].(string) + data.Response = wrapped.InternalEvent["data"].(string) + } + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data.MatcherName = k + results = append(results, &data) + } + } else { + results = append(results, &data) + } + return results +} diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go new file mode 100644 index 000000000..2b84b1a75 --- /dev/null +++ b/v2/pkg/protocols/network/request.go @@ -0,0 +1,96 @@ +package network + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +var _ protocols.Request = &Request{} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) { + address, err := getAddress(input) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not get address from url") + } + + // Compile each request for the template based on the URL + actualAddress, err := r.Make(address) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not build request") + } + + conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not connect to server request") + } + defer conn.Close() + conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + + _, err = conn.Write([]byte(r.Payload)) + if err != nil { + r.options.Output.Request(r.options.TemplateID, address, "network", err) + r.options.Progress.DecrementRequests(1) + return nil, errors.Wrap(err, "could not write request to server") + } + r.options.Progress.IncrementRequests() + + r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress) + + if r.options.Options.Debug { + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + fmt.Fprintf(os.Stderr, "%s\n", r.Payload) + } + + bufferSize := 1024 + if r.ReadSize != 0 { + bufferSize = r.ReadSize + } + buffer := make([]byte, bufferSize) + n, _ := conn.Read(buffer) + resp := string(buffer[:n]) + + if r.options.Options.Debug { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) + fmt.Fprintf(os.Stderr, "%s\n", resp) + } + ouputEvent := r.responseToDSLMap(r.Payload, resp, input, actualAddress) + + event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}} + if r.CompiledOperators != nil { + result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract) + if !ok { + return nil, nil + } + event[0].OperatorsResult = result + } + return event, nil +} + +// getAddress returns the address of the host to make request to +func getAddress(toTest string) (string, error) { + if strings.Contains(toTest, "://") { + parsed, err := url.Parse(toTest) + if err != nil { + return "", err + } + toTest = parsed.Host + } + return toTest, nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 8f18df1dc..e3f3e1468 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -32,12 +33,8 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { options.TemplateInfo = template.Info options.TemplatePath = file - // We don't support both http and dns in a single template - if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 { - return nil, fmt.Errorf("both http and dns requests for %s", template.ID) - } // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsNetwork)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -58,6 +55,9 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { for _, request := range template.RequestsHTTP { template.TotalRequests += request.Requests() } + for _, request := range template.RequestsNetwork { + template.TotalRequests += request.Requests() + } if len(template.RequestsDNS) > 0 { template.Executer = dns.NewExecuter(template.RequestsDNS, options) err = template.Executer.Compile() @@ -66,6 +66,10 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) { template.Executer = http.NewExecuter(template.RequestsHTTP, options) err = template.Executer.Compile() } + if len(template.RequestsNetwork) > 0 { + template.Executer = network.NewExecuter(template.RequestsNetwork, options) + err = template.Executer.Compile() + } if err != nil { return nil, errors.Wrap(err, "could not compile request") } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 8b54d9f0b..f4c341eaf 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -4,6 +4,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -17,6 +18,8 @@ type Template struct { RequestsHTTP []*http.Request `yaml:"requests,omitempty"` // RequestsDNS contains the dns request to make in the template RequestsDNS []*dns.Request `yaml:"dns,omitempty"` + // RequestsNetwork contains the network request to make in the template + RequestsNetwork []*network.Request `yaml:"network,omitempty"` // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline"` CompiledWorkflow *workflows.Workflow