Added network protocol support

v2/internal/runner/runner.go

@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
@@ -277,5 +278,8 @@ func (r *Runner) initializeProtocols() error {
 	if err := httpclientpool.Init(r.options); err != nil {
 		return err
 	}
+	if err := networkclientpool.Init(r.options); err != nil {
+		return err
+	}
 	return nil
 }

v2/pkg/protocols/network/executer.go

@@ -0,0 +1,90 @@
+package network
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+)
+
+// Executer executes a group of requests for a protocol
+type Executer struct {
+	requests []*Request
+	options  *protocols.ExecuterOptions
+}
+
+var _ protocols.Executer = &Executer{}
+
+// NewExecuter creates a new request executer for list of requests
+func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer {
+	return &Executer{requests: requests, options: options}
+}
+
+// Compile compiles the execution generators preparing any requests possible.
+func (e *Executer) Compile() error {
+	for _, request := range e.requests {
+		err := request.Compile(e.options)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Requests returns the total number of requests the rule will perform
+func (e *Executer) Requests() int {
+	var count int
+	for _, request := range e.requests {
+		count += int(request.Requests())
+	}
+	return count
+}
+
+// Execute executes the protocol group and returns true or false if results were found.
+func (e *Executer) Execute(input string) (bool, error) {
+	var results bool
+
+	for _, req := range e.requests {
+		events, err := req.ExecuteWithResults(input, nil)
+		if err != nil {
+			return false, err
+		}
+		if events == nil {
+			return false, nil
+		}
+
+		// If we have a result field, we should add a result to slice.
+		for _, event := range events {
+			if event.OperatorsResult == nil {
+				continue
+			}
+
+			for _, result := range req.makeResultEvent(event) {
+				results = true
+				e.options.Output.Write(result)
+			}
+		}
+	}
+	return results, nil
+}
+
+// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
+func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) {
+	var results []*output.InternalWrappedEvent
+
+	for _, req := range e.requests {
+		events, err := req.ExecuteWithResults(input, nil)
+		if err != nil {
+			return nil, err
+		}
+		if events == nil {
+			return nil, nil
+		}
+		for _, event := range events {
+			if event.OperatorsResult == nil {
+				continue
+			}
+			event.Results = req.makeResultEvent(event)
+		}
+		results = append(results, events...)
+	}
+	return results, nil
+}

v2/pkg/protocols/network/network.go

@@ -0,0 +1,79 @@
+package network
+
+import (
+	"net"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
+)
+
+// Request contains a Network protocol request to be made from a template
+type Request struct {
+	// Address is the address to send requests to (host:port combos generally)
+	Address     string `yaml:"address"`
+	addressHost string
+	addressPort string
+
+	// Payload is the payload to send for the network request
+	Payload string `yaml:"payload"`
+	// ReadSize is the size of response to read (1024 if not provided by default)
+	ReadSize int `yaml:"read-size"`
+
+	// Operators for the current request go here.
+	operators.Operators `yaml:",inline"`
+	CompiledOperators   *operators.Operators
+
+	// cache any variables that may be needed for operation.
+	dialer  *fastdialer.Dialer
+	options *protocols.ExecuterOptions
+}
+
+// Compile compiles the protocol request for further execution.
+func (r *Request) Compile(options *protocols.ExecuterOptions) error {
+	var err error
+	if strings.Contains(r.Address, ":") {
+		r.addressHost, r.addressPort, err = net.SplitHostPort(r.Address)
+		if err != nil {
+			return errors.Wrap(err, "could not parse address")
+		}
+	} else {
+		r.addressHost = r.Address
+	}
+
+	// Create a client for the class
+	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
+	if err != nil {
+		return errors.Wrap(err, "could not get network client")
+	}
+	r.dialer = client
+
+	if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
+		compiled := &r.Operators
+		if err := compiled.Compile(); err != nil {
+			return errors.Wrap(err, "could not compile operators")
+		}
+		r.CompiledOperators = compiled
+	}
+	r.options = options
+	return nil
+}
+
+// Requests returns the total number of requests the YAML rule will perform
+func (r *Request) Requests() int {
+	return 1
+}
+
+// Make returns the request to be sent for the protocol
+func (r *Request) Make(data string) (string, error) {
+	replacer := replacer.New(map[string]interface{}{"Address": data})
+	address := replacer.Replace(r.addressHost)
+	if !strings.Contains(address, ":") {
+		address = net.JoinHostPort(address, r.addressPort)
+	}
+	return address, nil
+}

v2/pkg/protocols/network/networkclientpool/clientpool.go

@@ -0,0 +1,38 @@
+package networkclientpool
+
+import (
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+var (
+	normalClient *fastdialer.Dialer
+)
+
+// Init initializes the clientpool implementation
+func Init(options *types.Options) error {
+	// Don't create clients if already created in past.
+	if normalClient != nil {
+		return nil
+	}
+	dialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
+	if err != nil {
+		return errors.Wrap(err, "could not create dialer")
+	}
+	normalClient = dialer
+	return nil
+}
+
+// Configuration contains the custom configuration options for a client
+type Configuration struct{}
+
+// Hash returns the hash of the configuration to allow client pooling
+func (c *Configuration) Hash() string {
+	return ""
+}
+
+// Get creates or gets a client for the protocol based on custom configuration
+func Get(options *types.Options, configuration *Configuration) (*fastdialer.Dialer, error) {
+	return normalClient, nil
+}

v2/pkg/protocols/network/operators.go

@@ -0,0 +1,108 @@
+package network
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Match matches a generic data response again a given matcher
+func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
+	partString := matcher.Part
+	switch partString {
+	case "body", "all", "":
+		partString = "data"
+	}
+
+	item, ok := data[partString]
+	if !ok {
+		return false
+	}
+	itemStr := types.ToString(item)
+
+	switch matcher.GetType() {
+	case matchers.SizeMatcher:
+		return matcher.Result(matcher.MatchSize(len(itemStr)))
+	case matchers.WordsMatcher:
+		return matcher.Result(matcher.MatchWords(itemStr))
+	case matchers.RegexMatcher:
+		return matcher.Result(matcher.MatchRegex(itemStr))
+	case matchers.BinaryMatcher:
+		return matcher.Result(matcher.MatchBinary(itemStr))
+	case matchers.DSLMatcher:
+		return matcher.Result(matcher.MatchDSL(data))
+	}
+	return false
+}
+
+// Extract performs extracting operation for a extractor on model and returns true or false.
+func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
+	part, ok := data[extractor.Part]
+	if !ok {
+		return nil
+	}
+	partString := part.(string)
+
+	switch partString {
+	case "body", "all":
+		partString = "data"
+	}
+
+	item, ok := data[partString]
+	if !ok {
+		return nil
+	}
+	itemStr := types.ToString(item)
+
+	switch extractor.GetType() {
+	case extractors.RegexExtractor:
+		return extractor.ExtractRegex(itemStr)
+	case extractors.KValExtractor:
+		return extractor.ExtractKval(data)
+	}
+	return nil
+}
+
+// responseToDSLMap converts a DNS response to a map for use in DSL matching
+func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent {
+	data := make(output.InternalEvent, 4)
+
+	// Some data regarding the request metadata
+	data["host"] = host
+	data["matched"] = matched
+	if r.options.Options.JSONRequests {
+		data["request"] = req
+	}
+	data["data"] = resp
+	return data
+}
+
+// makeResultEvent creates a result event from internal wrapped event
+func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
+	results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
+
+	data := output.ResultEvent{
+		TemplateID:       r.options.TemplateID,
+		Info:             r.options.TemplateInfo,
+		Type:             "network",
+		Host:             wrapped.InternalEvent["host"].(string),
+		Matched:          wrapped.InternalEvent["matched"].(string),
+		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
+	}
+	if r.options.Options.JSONRequests {
+		data.Request = wrapped.InternalEvent["request"].(string)
+		data.Response = wrapped.InternalEvent["data"].(string)
+	}
+
+	// If we have multiple matchers with names, write each of them separately.
+	if len(wrapped.OperatorsResult.Matches) > 0 {
+		for k := range wrapped.OperatorsResult.Matches {
+			data.MatcherName = k
+			results = append(results, &data)
+		}
+	} else {
+		results = append(results, &data)
+	}
+	return results
+}

v2/pkg/protocols/network/request.go

@@ -0,0 +1,96 @@
+package network
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+)
+
+var _ protocols.Request = &Request{}
+
+// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
+func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) {
+	address, err := getAddress(input)
+	if err != nil {
+		r.options.Output.Request(r.options.TemplateID, input, "network", err)
+		r.options.Progress.DecrementRequests(1)
+		return nil, errors.Wrap(err, "could not get address from url")
+	}
+
+	// Compile each request for the template based on the URL
+	actualAddress, err := r.Make(address)
+	if err != nil {
+		r.options.Output.Request(r.options.TemplateID, address, "network", err)
+		r.options.Progress.DecrementRequests(1)
+		return nil, errors.Wrap(err, "could not build request")
+	}
+
+	conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress)
+	if err != nil {
+		r.options.Output.Request(r.options.TemplateID, address, "network", err)
+		r.options.Progress.DecrementRequests(1)
+		return nil, errors.Wrap(err, "could not connect to server request")
+	}
+	defer conn.Close()
+	conn.SetReadDeadline(time.Now().Add(5 * time.Second))
+
+	_, err = conn.Write([]byte(r.Payload))
+	if err != nil {
+		r.options.Output.Request(r.options.TemplateID, address, "network", err)
+		r.options.Progress.DecrementRequests(1)
+		return nil, errors.Wrap(err, "could not write request to server")
+	}
+	r.options.Progress.IncrementRequests()
+
+	r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err)
+	gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress)
+
+	if r.options.Options.Debug {
+		gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress)
+		fmt.Fprintf(os.Stderr, "%s\n", r.Payload)
+	}
+
+	bufferSize := 1024
+	if r.ReadSize != 0 {
+		bufferSize = r.ReadSize
+	}
+	buffer := make([]byte, bufferSize)
+	n, _ := conn.Read(buffer)
+	resp := string(buffer[:n])
+
+	if r.options.Options.Debug {
+		gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
+		fmt.Fprintf(os.Stderr, "%s\n", resp)
+	}
+	ouputEvent := r.responseToDSLMap(r.Payload, resp, input, actualAddress)
+
+	event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}}
+	if r.CompiledOperators != nil {
+		result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract)
+		if !ok {
+			return nil, nil
+		}
+		event[0].OperatorsResult = result
+	}
+	return event, nil
+}
+
+// getAddress returns the address of the host to make request to
+func getAddress(toTest string) (string, error) {
+	if strings.Contains(toTest, "://") {
+		parsed, err := url.Parse(toTest)
+		if err != nil {
+			return "", err
+		}
+		toTest = parsed.Host
+	}
+	return toTest, nil
+}

v2/pkg/templates/compile.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"gopkg.in/yaml.v2"
 )
@@ -32,12 +33,8 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
 	options.TemplateInfo = template.Info
 	options.TemplatePath = file
 
-	// We don't support both http and dns in a single template
-	if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 {
-		return nil, fmt.Errorf("both http and dns requests for %s", template.ID)
-	}
 	// If no requests, and it is also not a workflow, return error.
-	if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 {
+	if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsNetwork)+len(template.Workflows) == 0 {
 		return nil, fmt.Errorf("no requests defined for %s", template.ID)
 	}
 
@@ -58,6 +55,9 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
 	for _, request := range template.RequestsHTTP {
 		template.TotalRequests += request.Requests()
 	}
+	for _, request := range template.RequestsNetwork {
+		template.TotalRequests += request.Requests()
+	}
 	if len(template.RequestsDNS) > 0 {
 		template.Executer = dns.NewExecuter(template.RequestsDNS, options)
 		err = template.Executer.Compile()
@@ -66,6 +66,10 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
 		template.Executer = http.NewExecuter(template.RequestsHTTP, options)
 		err = template.Executer.Compile()
 	}
+	if len(template.RequestsNetwork) > 0 {
+		template.Executer = network.NewExecuter(template.RequestsNetwork, options)
+		err = template.Executer.Compile()
+	}
 	if err != nil {
 		return nil, errors.Wrap(err, "could not compile request")
 	}

v2/pkg/templates/templates.go

@@ -4,6 +4,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 )
 
@@ -17,6 +18,8 @@ type Template struct {
 	RequestsHTTP []*http.Request `yaml:"requests,omitempty"`
 	// RequestsDNS contains the dns request to make in the template
 	RequestsDNS []*dns.Request `yaml:"dns,omitempty"`
+	// RequestsNetwork contains the network request to make in the template
+	RequestsNetwork []*network.Request `yaml:"network,omitempty"`
 	// Workflows is a yaml based workflow declaration code.
 	workflows.Workflow `yaml:",inline"`
 	CompiledWorkflow   *workflows.Workflow