Whois Protocol Support (using rdap) (#1354)

* init rdap

* add an integration test, option to supply RDAP server to execute the request on

* add rdap protocolMappings

* add debug info, add IP, ASN query type support

* rename rdap to whois, Host to Query in template

* rename pending rdap to whois

* remove port from whois varaiables

* set Host variable even if input is not a parsable url

integration_tests/whois/basic.yaml

@@ -0,0 +1,14 @@
+id: basic-whois-example
+
+info:
+  name: test template for WHOIS
+  author: pdteam
+  severity: info
+
+whois:
+  - query: "{{Host}}"
+    extractors:
+      - type: kval
+        kval:
+          - "expiration date"
+          - "registrar"

v2/cmd/integration-test/integration-test.go

@@ -26,6 +26,7 @@ var (
 		"loader":    loaderTestcases,
 		"websocket": websocketTestCases,
 		"headless":  headlessTestcases,
+		"whois":     whoisTestCases,
 	}
 )
 

v2/cmd/integration-test/whois.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+)
+
+var whoisTestCases = map[string]testutils.TestCase{
+	"whois/basic.yaml": &whoisBasic{},
+}
+
+type whoisBasic struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *whoisBasic) Execute(filePath string) error {
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

v2/go.mod

@@ -75,6 +75,8 @@ require (
 	github.com/PuerkitoBio/goquery v1.6.0 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/andybalholm/cascadia v1.1.0 // indirect
 	github.com/antchfx/xpath v1.2.0 // indirect
@@ -111,8 +113,10 @@ require (
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/openrdap/rdap v0.9.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
 	github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect
@@ -133,6 +137,7 @@ require (
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

v2/go.sum

@@ -71,8 +71,10 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n
 github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA=
 github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
@@ -501,6 +503,7 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
@@ -551,6 +554,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7
 github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8=
 github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/openrdap/rdap v0.9.0 h1:qaicAQD7XLPH8k4B8FQ6OnBL10joHt2xTREFA38T+wA=
+github.com/openrdap/rdap v0.9.0/go.mod h1:Z9b01CBKIgQsX1JzmmhkhoQfdWYchgCP6B4QfKXSdD0=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -1209,6 +1214,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

v2/pkg/operators/matchers/match.go

@@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {
 
 // MatchDSL matches on a generic map result
 func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {
-	logExpressionEvaluationFailure := func (matcherName string, err error) {
+	logExpressionEvaluationFailure := func(matcherName string, err error) {
 		gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error())
 	}
 

v2/pkg/protocols/whois/whois.go

@@ -0,0 +1,192 @@
+package whois
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/openrdap/rdap"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
+	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
+	"github.com/projectdiscovery/nuclei/v2/pkg/types"
+)
+
+// Request is a request for the WHOIS protocol
+type Request struct {
+	// Operators for the current request go here.
+	operators.Operators `yaml:",inline,omitempty"`
+	CompiledOperators   *operators.Operators `yaml:"-"`
+
+	// description: |
+	//   Query contains query for the request
+	Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"`
+
+	// description: |
+	// 	 Optional WHOIS server URL.
+	//
+	// 	 If present, specifies the WHOIS server to execute the Request on.
+	//   Otherwise, nil enables bootstrapping
+	Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"`
+	// cache any variables that may be needed for operation.
+	client          *rdap.Client
+	options         *protocols.ExecuterOptions
+	parsedServerURL *url.URL
+}
+
+// Compile compiles the request generators preparing any requests possible.
+func (request *Request) Compile(options *protocols.ExecuterOptions) error {
+	var err error
+	if request.Server != "" {
+		request.parsedServerURL, err = url.Parse(request.Server)
+		if err != nil {
+			return errors.Wrap(err, "failed to parse server URL")
+		}
+	}
+
+	request.options = options
+	request.client = &rdap.Client{}
+
+	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
+		compiled := &request.Operators
+		if err := compiled.Compile(); err != nil {
+			return errors.Wrap(err, "could not compile operators")
+		}
+		request.CompiledOperators = compiled
+	}
+	return nil
+}
+
+// Requests returns the total number of requests the rule will perform
+func (request *Request) Requests() int {
+	return 1
+}
+
+// GetID returns the ID for the request if any.
+func (request *Request) GetID() string {
+	return ""
+}
+
+// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
+func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	// generate variables
+	variables := generateVariables(input)
+	// and replace placeholders
+	query := replacer.Replace(request.Query, variables)
+	// build an rdap request
+	rdapReq := rdap.NewAutoRequest(query)
+	res, err := request.client.Do(rdapReq)
+	if err != nil {
+		return errors.Wrap(err, "could not make whois request")
+	}
+	gologger.Verbose().Msgf("Sent WHOIS request to %s", query)
+	if request.options.Options.Debug || request.options.Options.DebugRequests {
+		gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query)
+	}
+
+	data := make(map[string]interface{})
+	var response interface{}
+	switch rdapReq.Type {
+	case rdap.DomainRequest:
+		// convert the rdap response to a whois style response (for domain request type only)
+		whoisResp := res.ToWhoisStyleResponse()
+		for k, v := range whoisResp.Data {
+			data[strings.ToLower(k)] = strings.Join(v, ",")
+		}
+		response = whoisResp
+	default:
+		response = res.Object
+	}
+	jsonData, _ := jsoniter.Marshal(response)
+	jsonDataString := string(jsonData)
+
+	data["type"] = request.Type().String()
+	data["host"] = query
+	data["response"] = jsonDataString
+
+	event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
+	if request.options.Options.Debug || request.options.Options.DebugResponse {
+		gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)
+		gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))
+	}
+
+	callback(event)
+	return nil
+}
+
+// Match performs matching operation for a matcher on model and returns:
+// true and a list of matched snippets if the matcher type is supports it
+// otherwise false and an empty string slice
+func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
+	return protocols.MakeDefaultMatchFunc(data, matcher)
+}
+
+// Extract performs extracting operation for an extractor on model and returns true or false.
+func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
+	return protocols.MakeDefaultExtractFunc(data, matcher)
+}
+
+// MakeResultEvent creates a result event from internal wrapped event
+func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
+	return protocols.MakeDefaultResultEvent(request, wrapped)
+}
+
+// GetCompiledOperators returns a list of the compiled operators
+func (request *Request) GetCompiledOperators() []*operators.Operators {
+	return []*operators.Operators{request.CompiledOperators}
+}
+
+func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
+	data := &output.ResultEvent{
+		TemplateID:       types.ToString(request.options.TemplateID),
+		TemplatePath:     types.ToString(request.options.TemplatePath),
+		Info:             request.options.TemplateInfo,
+		Type:             types.ToString(wrapped.InternalEvent["type"]),
+		Host:             types.ToString(wrapped.InternalEvent["host"]),
+		Metadata:         wrapped.OperatorsResult.PayloadValues,
+		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
+		Timestamp:        time.Now(),
+		MatcherStatus:    true,
+		Request:          types.ToString(wrapped.InternalEvent["request"]),
+		Response:         types.ToString(wrapped.InternalEvent["response"]),
+	}
+	return data
+}
+
+// Type returns the type of the protocol request
+func (request *Request) Type() templateTypes.ProtocolType {
+	return templateTypes.WHOISProtocol
+}
+
+// generateVariables will create default variables after parsing a url
+func generateVariables(input string) map[string]interface{} {
+	var domain string
+
+	parsed, err := url.Parse(input)
+	if err != nil {
+		return map[string]interface{}{"Input": input}
+	}
+	domain = parsed.Host
+	if domain == "" {
+		domain = input
+	}
+	if strings.Contains(domain, ":") {
+		domain = strings.Split(domain, ":")[0]
+	}
+
+	return map[string]interface{}{
+		"Input":    input,
+		"Hostname": parsed.Host,
+		"Host":     domain,
+	}
+}

v2/pkg/templates/compile.go

@@ -128,7 +128,8 @@ func (template *Template) Requests() int {
 		len(template.RequestsHeadless) +
 		len(template.Workflows) +
 		len(template.RequestsSSL) +
-		len(template.RequestsWebsocket)
+		len(template.RequestsWebsocket) +
+		len(template.RequestsWHOIS)
 }
 
 // compileProtocolRequests compiles all the protocol requests for the template
@@ -166,6 +167,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti
 
 	case len(template.RequestsWebsocket) > 0:
 		requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket)
+
+	case len(template.RequestsWHOIS) > 0:
+		requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS)
 	}
 	template.Executer = executer.NewExecuter(requests, &options)
 	return nil

v2/pkg/templates/templates.go

@@ -14,6 +14,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
 	"go.uber.org/multierr"
@@ -71,6 +72,9 @@ type Template struct {
 	//   Websocket contains the Websocket request to make in the template.
 	RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"`
 
+	// description: |
+	//   WHOIS contains the WHOIS request to make in the template.
+	RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"`
 	// description: |
 	//   Workflows is a yaml based workflow declaration code.
 	workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"`
@@ -101,6 +105,7 @@ var TemplateProtocols = []string{
 	"workflow",
 	"ssl",
 	"websocket",
+	"whois",
 }
 
 // Type returns the type of the template
@@ -122,6 +127,8 @@ func (template *Template) Type() types.ProtocolType {
 		return types.SSLProtocol
 	case len(template.RequestsWebsocket) > 0:
 		return types.WebsocketProtocol
+	case len(template.RequestsWHOIS) > 0:
+		return types.WHOISProtocol
 	default:
 		return types.InvalidProtocol
 	}

v2/pkg/templates/types/types.go

@@ -34,6 +34,8 @@ const (
 	SSLProtocol
 	// name:websocket
 	WebsocketProtocol
+	// name:whois
+	WHOISProtocol
 	limit
 	InvalidProtocol
 )
@@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{
 	WorkflowProtocol:  "workflow",
 	SSLProtocol:       "ssl",
 	WebsocketProtocol: "websocket",
+	WHOISProtocol:     "whois",
 }
 
 func GetSupportedProtocolTypes() ProtocolTypes {