From 5200bcd94f7ca146ea5c1408965762b883cf847d Mon Sep 17 00:00:00 2001 From: Sajad Date: Thu, 16 Dec 2021 17:08:02 +0530 Subject: [PATCH] Whois Protocol Support (using rdap) (#1354) * init rdap * add an integration test, option to supply RDAP server to execute the request on * add rdap protocolMappings * add debug info, add IP, ASN query type support * rename rdap to whois, Host to Query in template * rename pending rdap to whois * remove port from whois varaiables * set Host variable even if input is not a parsable url --- integration_tests/whois/basic.yaml | 14 ++ v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/whois.go | 23 +++ v2/go.mod | 5 + v2/go.sum | 6 + v2/pkg/operators/matchers/match.go | 2 +- v2/pkg/protocols/whois/whois.go | 192 ++++++++++++++++++++ v2/pkg/templates/compile.go | 6 +- v2/pkg/templates/templates.go | 7 + v2/pkg/templates/types/types.go | 3 + 10 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 integration_tests/whois/basic.yaml create mode 100644 v2/cmd/integration-test/whois.go create mode 100644 v2/pkg/protocols/whois/whois.go diff --git a/integration_tests/whois/basic.yaml b/integration_tests/whois/basic.yaml new file mode 100644 index 000000000..742272a9a --- /dev/null +++ b/integration_tests/whois/basic.yaml @@ -0,0 +1,14 @@ +id: basic-whois-example + +info: + name: test template for WHOIS + author: pdteam + severity: info + +whois: + - query: "{{Host}}" + extractors: + - type: kval + kval: + - "expiration date" + - "registrar" \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index be9ab4c16..4bbed10a1 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -26,6 +26,7 @@ var ( "loader": loaderTestcases, "websocket": websocketTestCases, "headless": headlessTestcases, + "whois": whoisTestCases, } ) diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go new file mode 100644 index 000000000..3935f40e8 --- /dev/null +++ b/v2/cmd/integration-test/whois.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var whoisTestCases = map[string]testutils.TestCase{ + "whois/basic.yaml": &whoisBasic{}, +} + +type whoisBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *whoisBasic) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/go.mod b/v2/go.mod index 41dbc4d87..2d0a8d9c6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -75,6 +75,8 @@ require ( github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.2.0 // indirect @@ -111,8 +113,10 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/openrdap/rdap v0.9.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect @@ -133,6 +137,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 3df87760e..f53abb112 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -71,8 +71,10 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= @@ -501,6 +503,7 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -551,6 +554,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7 github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/openrdap/rdap v0.9.0 h1:qaicAQD7XLPH8k4B8FQ6OnBL10joHt2xTREFA38T+wA= +github.com/openrdap/rdap v0.9.0/go.mod h1:Z9b01CBKIgQsX1JzmmhkhoQfdWYchgCP6B4QfKXSdD0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1209,6 +1214,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 898eaca0a..052e2c856 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { // MatchDSL matches on a generic map result func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { - logExpressionEvaluationFailure := func (matcherName string, err error) { + logExpressionEvaluationFailure := func(matcherName string, err error) { gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error()) } diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go new file mode 100644 index 000000000..6da0305f5 --- /dev/null +++ b/v2/pkg/protocols/whois/whois.go @@ -0,0 +1,192 @@ +package whois + +import ( + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/openrdap/rdap" + "github.com/pkg/errors" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the WHOIS protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Query contains query for the request + Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` + + // description: | + // Optional WHOIS server URL. + // + // If present, specifies the WHOIS server to execute the Request on. + // Otherwise, nil enables bootstrapping + Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"` + // cache any variables that may be needed for operation. + client *rdap.Client + options *protocols.ExecuterOptions + parsedServerURL *url.URL +} + +// Compile compiles the request generators preparing any requests possible. +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + var err error + if request.Server != "" { + request.parsedServerURL, err = url.Parse(request.Server) + if err != nil { + return errors.Wrap(err, "failed to parse server URL") + } + } + + request.options = options + request.client = &rdap.Client{} + + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + request.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (request *Request) Requests() int { + return 1 +} + +// GetID returns the ID for the request if any. +func (request *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + // generate variables + variables := generateVariables(input) + // and replace placeholders + query := replacer.Replace(request.Query, variables) + // build an rdap request + rdapReq := rdap.NewAutoRequest(query) + res, err := request.client.Do(rdapReq) + if err != nil { + return errors.Wrap(err, "could not make whois request") + } + gologger.Verbose().Msgf("Sent WHOIS request to %s", query) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query) + } + + data := make(map[string]interface{}) + var response interface{} + switch rdapReq.Type { + case rdap.DomainRequest: + // convert the rdap response to a whois style response (for domain request type only) + whoisResp := res.ToWhoisStyleResponse() + for k, v := range whoisResp.Data { + data[strings.ToLower(k)] = strings.Join(v, ",") + } + response = whoisResp + default: + response = res.Object + } + jsonData, _ := jsoniter.Marshal(response) + jsonDataString := string(jsonData) + + data["type"] = request.Type().String() + data["host"] = query + data["response"] = jsonDataString + + event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) + } + + callback(event) + return nil +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, + Type: types.ToString(wrapped.InternalEvent["type"]), + Host: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + MatcherStatus: true, + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["response"]), + } + return data +} + +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.WHOISProtocol +} + +// generateVariables will create default variables after parsing a url +func generateVariables(input string) map[string]interface{} { + var domain string + + parsed, err := url.Parse(input) + if err != nil { + return map[string]interface{}{"Input": input} + } + domain = parsed.Host + if domain == "" { + domain = input + } + if strings.Contains(domain, ":") { + domain = strings.Split(domain, ":")[0] + } + + return map[string]interface{}{ + "Input": input, + "Hostname": parsed.Host, + "Host": domain, + } +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 079c96f4a..84c56acd8 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -128,7 +128,8 @@ func (template *Template) Requests() int { len(template.RequestsHeadless) + len(template.Workflows) + len(template.RequestsSSL) + - len(template.RequestsWebsocket) + len(template.RequestsWebsocket) + + len(template.RequestsWHOIS) } // compileProtocolRequests compiles all the protocol requests for the template @@ -166,6 +167,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti case len(template.RequestsWebsocket) > 0: requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket) + + case len(template.RequestsWHOIS) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS) } template.Executer = executer.NewExecuter(requests, &options) return nil diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index bdf595cf0..d8bacffd8 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "go.uber.org/multierr" @@ -71,6 +72,9 @@ type Template struct { // Websocket contains the Websocket request to make in the template. RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` + // description: | + // WHOIS contains the WHOIS request to make in the template. + RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"` @@ -101,6 +105,7 @@ var TemplateProtocols = []string{ "workflow", "ssl", "websocket", + "whois", } // Type returns the type of the template @@ -122,6 +127,8 @@ func (template *Template) Type() types.ProtocolType { return types.SSLProtocol case len(template.RequestsWebsocket) > 0: return types.WebsocketProtocol + case len(template.RequestsWHOIS) > 0: + return types.WHOISProtocol default: return types.InvalidProtocol } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index be6b72453..cc0c8b93d 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -34,6 +34,8 @@ const ( SSLProtocol // name:websocket WebsocketProtocol + // name:whois + WHOISProtocol limit InvalidProtocol ) @@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{ WorkflowProtocol: "workflow", SSLProtocol: "ssl", WebsocketProtocol: "websocket", + WHOISProtocol: "whois", } func GetSupportedProtocolTypes() ProtocolTypes {