diff --git a/internal/runner/options.go b/internal/runner/options.go index 91eb0988a..53321dcd4 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -7,15 +7,6 @@ import ( "github.com/projectdiscovery/gologger" ) -// DefaultResolvers contains the list of resolvers known to be trusted. -var DefaultResolvers = []string{ - "1.1.1.1:53", // Cloudflare - "1.0.0.1:53", // Cloudflare - "8.8.8.8:53", // Google - "8.8.4.4:53", // Google - "9.9.9.9:53", // Quad9 -} - // Options contains the configuration options for tuning // the template requesting process. type Options struct { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index ea87a5e52..f298ee51c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -165,59 +165,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i func (r *Runner) sendRequest(template *templates.Template, request interface{}, URL string, writer *bufio.Writer, httpclient *retryablehttp.Client, dnsclient *retryabledns.Client) { switch request.(type) { case *requests.HTTPRequest: - if !isURL(URL) { - break - } - httpRequest := request.(*requests.HTTPRequest) - - // Compile each request for the template based on the URL - compiledRequest, err := httpRequest.MakeHTTPRequest(URL) - if err != nil { - gologger.Warningf("[%s] Could not make request %s: %s\n", template.ID, URL, err) - return - } - - // Send the request to the target servers - for _, req := range compiledRequest { - resp, err := httpclient.Do(req) - if err != nil { - if resp != nil { - resp.Body.Close() - } - gologger.Warningf("[%s] Could not send request %s: %s\n", template.ID, URL, err) - return - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - gologger.Warningf("[%s] Could not read body %s: %s\n", template.ID, URL, err) - continue - } - resp.Body.Close() - - body := unsafeToString(data) - - var headers string - for _, matcher := range httpRequest.Matchers { - // Only build the headers string if the matcher asks for it - part := matcher.GetPart() - if part == matchers.AllPart || part == matchers.HeaderPart && headers == "" { - headers = headersToString(resp.Header) - } - - // Check if the matcher matched - if matcher.Match(resp, body, headers) { - // If there is an extractor, run it. - var extractorResults []string - for _, extractor := range httpRequest.Extractors { - part := extractor.GetPart() - if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" { - headers = headersToString(resp.Header) - } - extractorResults = append(extractorResults, extractor.Extract(body, headers)...) - } + // All the matchers matched, print the output on the screen output := buildOutputHTTP(template, req, extractorResults, matcher) @@ -232,40 +180,6 @@ func (r *Runner) sendRequest(template *templates.Template, request interface{}, } } case *requests.DNSRequest: - // eventually extracts dns from url - var domain string = URL - if isURL(URL) { - domain = extractDomain(URL) - } - - dnsRequest := request.(*requests.DNSRequest) - - // Compile each request for the template based on the URL - compiledRequest, err := dnsRequest.MakeDNSRequest(domain) - if err != nil { - gologger.Warningf("[%s] Could not make request %s: %s\n", template.ID, domain, err) - return - } - - // Send the request to the target servers - resp, err := dnsclient.Do(compiledRequest) - if err != nil { - gologger.Warningf("[%s] Could not send request %s: %s\n", template.ID, domain, err) - return - } - - for _, matcher := range dnsRequest.Matchers { - // Check if the matcher matched - if !matcher.MatchDNS(resp) { - return - } - } - - // If there is an extractor, run it. - var extractorResults []string - for _, extractor := range dnsRequest.Extractors { - extractorResults = append(extractorResults, extractor.ExtractDNS(resp.String())...) - } // All the matchers matched, print the output on the screen output := buildOutputDNS(template, domain, extractorResults) @@ -278,117 +192,3 @@ func (r *Runner) sendRequest(template *templates.Template, request interface{}, } } } - -// buildOutputHTTP builds an output text for writing results -func buildOutputHTTP(template *templates.Template, req *retryablehttp.Request, extractorResults []string, matcher *matchers.Matcher) string { - builder := &strings.Builder{} - builder.WriteRune('[') - builder.WriteString(template.ID) - if len(matcher.Name) > 0 { - builder.WriteString(":") - builder.WriteString(matcher.Name) - } - builder.WriteString("] ") - - // Escape the URL by replacing all % with %% - URL := req.URL.String() - escapedURL := strings.Replace(URL, "%", "%%", -1) - builder.WriteString(escapedURL) - - // If any extractors, write the results - if len(extractorResults) > 0 { - builder.WriteString(" [") - for i, result := range extractorResults { - builder.WriteString(result) - if i != len(extractorResults)-1 { - builder.WriteRune(',') - } - } - builder.WriteString("]") - } - builder.WriteRune('\n') - - return builder.String() -} - -// buildOutput builds an output text for writing results -func buildOutputDNS(template *templates.Template, domain string, extractorResults []string) string { - builder := &strings.Builder{} - builder.WriteRune('[') - builder.WriteString(template.ID) - builder.WriteString("] [dns] ") - - builder.WriteString(domain) - - // If any extractors, write the results - if len(extractorResults) > 0 { - builder.WriteString(" [") - for i, result := range extractorResults { - builder.WriteString(result) - if i != len(extractorResults)-1 { - builder.WriteRune(',') - } - } - builder.WriteString("]") - } - builder.WriteRune('\n') - - return builder.String() -} - -// makeHTTPClient creates a HTTP client with configurable redirect field -func (r *Runner) makeHTTPClientByRequest(request interface{}) *retryablehttp.Client { - - redirects := false - maxRedirects := 0 - // Request is HTTP - if httpRequest, ok := request.(requests.HTTPRequest); ok { - redirects = httpRequest.Redirects - maxRedirects = httpRequest.MaxRedirects - } - - retryablehttpOptions := retryablehttp.DefaultOptionsSpraying - retryablehttpOptions.RetryWaitMax = 10 * time.Second - retryablehttpOptions.RetryMax = r.options.Retries - - // Create the HTTP Client - client := retryablehttp.NewWithHTTPClient(&http.Client{ - Transport: &http.Transport{ - MaxIdleConnsPerHost: -1, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - }, - Timeout: time.Duration(r.options.Timeout) * time.Second, - CheckRedirect: func(_ *http.Request, requests []*http.Request) error { - if !redirects { - return http.ErrUseLastResponse - } - if maxRedirects == 0 { - if len(requests) > 10 { - return http.ErrUseLastResponse - } - return nil - } - if len(requests) > maxRedirects { - return http.ErrUseLastResponse - } - return nil - }, - }, retryablehttpOptions) - client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - return client -} - -// makeHTTPClient creates a HTTP client with configurable redirect field -func (r *Runner) makeDNSClientByRequest(request interface{}) *retryabledns.Client { - retries := r.options.Retries - if dnsRequest, ok := request.(*requests.DNSRequest); ok { - retries = dnsRequest.Retries - } - - dnsClient, _ := retryabledns.New(DefaultResolvers, retries) - return dnsClient -} diff --git a/internal/runner/utils.go b/pkg/executor/dns_utils.go similarity index 52% rename from internal/runner/utils.go rename to pkg/executor/dns_utils.go index 54bd53730..6009e5011 100644 --- a/internal/runner/utils.go +++ b/pkg/executor/dns_utils.go @@ -1,38 +1,11 @@ -package runner +package executor import ( - "net/http" "net/url" - "strings" - "unsafe" "github.com/asaskevich/govalidator" ) -// unsafeToString converts byte slice to string with zero allocations -func unsafeToString(bs []byte) string { - return *(*string)(unsafe.Pointer(&bs)) -} - -// headersToString converts http headers to string -func headersToString(headers http.Header) string { - builder := &strings.Builder{} - - for header, values := range headers { - builder.WriteString(header) - builder.WriteString(": ") - - for i, value := range values { - builder.WriteString(value) - if i != len(values)-1 { - builder.WriteRune(',') - } - } - builder.WriteRune('\n') - } - return builder.String() -} - // isURL tests a string to determine if it is a well-structured url or not. func isURL(toTest string) bool { _, err := url.ParseRequestURI(toTest) @@ -44,17 +17,17 @@ func isURL(toTest string) bool { if err != nil || u.Scheme == "" || u.Host == "" { return false } - return true } +// extractDomain extracts the domain name of a URL func extractDomain(URL string) string { u, err := url.Parse(URL) if err != nil { return "" } - - return u.Hostname() + hostname := u.Hostname() + return hostname } // isDNS tests a string to determine if it is a well-structured dns or not diff --git a/pkg/executor/executer_http.go b/pkg/executor/executer_http.go new file mode 100644 index 000000000..817946202 --- /dev/null +++ b/pkg/executor/executer_http.go @@ -0,0 +1,127 @@ +package executor + +import ( + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/pkg/extractors" + "github.com/projectdiscovery/nuclei/pkg/matchers" + "github.com/projectdiscovery/nuclei/pkg/requests" + "github.com/projectdiscovery/nuclei/pkg/templates" + "github.com/projectdiscovery/retryablehttp-go" +) + +// HTTPExecutor is client for performing HTTP requests +// for a template. +type HTTPExecutor struct { + httpClient *retryablehttp.Client + template *templates.Template + httpRequest *requests.HTTPRequest +} + +// NewHTTPExecutor creates a new HTTP executor from a template +// and a HTTP request query. +func NewHTTPExecutor(template *templates.Template, httpRequest *requests.HTTPRequest) *HTTPExecutor { + retryablehttpOptions := retryablehttp.DefaultOptionsSpraying + retryablehttpOptions.RetryWaitMax = 10 * time.Second + retryablehttpOptions.RetryMax = r.options.Retries + + // Create the HTTP Client + client := retryablehttp.NewWithHTTPClient(&http.Client{ + Transport: &http.Transport{ + MaxIdleConnsPerHost: -1, + TLSClientConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + }, + Timeout: time.Duration(r.options.Timeout) * time.Second, + CheckRedirect: func(_ *http.Request, requests []*http.Request) error { + if !httpRequest.Redirects { + return http.ErrUseLastResponse + } + if httpRequest.MaxRedirects == 0 { + if len(requests) > 10 { + return http.ErrUseLastResponse + } + return nil + } + if len(requests) > httpRequest.MaxRedirects { + return http.ErrUseLastResponse + } + return nil + }, + }, retryablehttpOptions) + client.CheckRetry = retryablehttp.HostSprayRetryPolicy() + + executer := &HTTPExecutor{ + httpClient: client, + template: template, + httpRequest: httpRequest, + } + return executer +} + +// ExecuteHTTP executes the HTTP request on a URL +func (e *HTTPExecutor) ExecuteHTTP(URL string) { + if !isURL(URL) { + return + } + + // Compile each request for the template based on the URL + compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL) + if err != nil { + gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, URL, err) + return + } + + // Send the request to the target servers + for _, req := range compiledRequest { + resp, err := e.httpClient.Do(req) + if err != nil { + if resp != nil { + resp.Body.Close() + } + gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, URL, err) + return + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + gologger.Warningf("[%s] Could not read body %s: %s\n", e.template.ID, URL, err) + continue + } + resp.Body.Close() + + body := unsafeToString(data) + + var headers string + for _, matcher := range e.httpRequest.Matchers { + // Only build the headers string if the matcher asks for it + part := matcher.GetPart() + if part == matchers.AllPart || part == matchers.HeaderPart && headers == "" { + headers = headersToString(resp.Header) + } + + // Check if the matcher matched + if matcher.Match(resp, body, headers) { + // If there is an extractor, run it. + var extractorResults []string + for _, extractor := range e.httpRequest.Extractors { + part := extractor.GetPart() + if part == extractors.AllPart || part == extractors.HeaderPart && headers == "" { + headers = headersToString(resp.Header) + } + extractorResults = append(extractorResults, extractor.Extract(body, headers)...) + } + } + } + } +} diff --git a/pkg/executor/executor_dns.go b/pkg/executor/executor_dns.go new file mode 100644 index 000000000..c4438c049 --- /dev/null +++ b/pkg/executor/executor_dns.go @@ -0,0 +1,75 @@ +package executor + +import ( + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/pkg/requests" + "github.com/projectdiscovery/nuclei/pkg/templates" + retryabledns "github.com/projectdiscovery/retryabledns" +) + +// DNSExecutor is a client for performing a DNS request +// for a template. +type DNSExecutor struct { + dnsClient *retryabledns.Client + template *templates.Template + dnsRequest *requests.DNSRequest +} + +// DefaultResolvers contains the list of resolvers known to be trusted. +var DefaultResolvers = []string{ + "1.1.1.1:53", // Cloudflare + "1.0.0.1:53", // Cloudflare + "8.8.8.8:53", // Google + "8.8.4.4:53", // Google +} + +// NewDNSExecutor creates a new DNS executor from a template +// and a DNS request query. +func NewDNSExecutor(template *templates.Template, dnsRequest *requests.DNSRequest) *DNSExecutor { + dnsClient := retryabledns.New(DefaultResolvers, dnsRequest.Retries) + + executer := &DNSExecutor{ + dnsClient: dnsClient, + template: template, + dnsRequest: dnsRequest, + } + return executer +} + +// ExecuteDNS executes the DNS request on a URL +func (e *DNSExecutor) ExecuteDNS(URL string) { + // Parse the URL and return domain if URL. + var domain string + if isURL(URL) { + domain = extractDomain(URL) + } else { + domain = URL + } + + // Compile each request for the template based on the URL + compiledRequest, err := e.dnsRequest.MakeDNSRequest(URL) + if err != nil { + gologger.Warningf("[%s] Could not make request %s: %s\n", e.template.ID, domain, err) + return + } + + // Send the request to the target servers + resp, err := e.dnsClient.Do(compiledRequest) + if err != nil { + gologger.Warningf("[%s] Could not send request %s: %s\n", e.template.ID, domain, err) + return + } + + for _, matcher := range e.dnsRequest.Matchers { + // Check if the matcher matched + if !matcher.MatchDNS(resp) { + return + } + } + + // If there is an extractor, run it. + var extractorResults []string + for _, extractor := range e.dnsRequest.Extractors { + extractorResults = append(extractorResults, extractor.ExtractDNS(resp.String())...) + } +} diff --git a/pkg/executor/http_utils.go b/pkg/executor/http_utils.go new file mode 100644 index 000000000..eafcc3969 --- /dev/null +++ b/pkg/executor/http_utils.go @@ -0,0 +1,31 @@ +package executor + +import ( + "net/http" + "strings" + "unsafe" +) + +// unsafeToString converts byte slice to string with zero allocations +func unsafeToString(bs []byte) string { + return *(*string)(unsafe.Pointer(&bs)) +} + +// headersToString converts http headers to string +func headersToString(headers http.Header) string { + builder := &strings.Builder{} + + for header, values := range headers { + builder.WriteString(header) + builder.WriteString(": ") + + for i, value := range values { + builder.WriteString(value) + if i != len(values)-1 { + builder.WriteRune(',') + } + } + builder.WriteRune('\n') + } + return builder.String() +} diff --git a/pkg/executor/output_dns.go b/pkg/executor/output_dns.go new file mode 100644 index 000000000..facafdc53 --- /dev/null +++ b/pkg/executor/output_dns.go @@ -0,0 +1,30 @@ +package executor + +import ( + "strings" +) + +// buildOutput builds an output text for writing results +func (e *DNSExecutor) buildOutputDNS(domain string, extractorResults []string) string { + builder := &strings.Builder{} + builder.WriteRune('[') + builder.WriteString(e.template.ID) + builder.WriteString("] [dns] ") + + builder.WriteString(domain) + + // If any extractors, write the results + if len(extractorResults) > 0 { + builder.WriteString(" [") + for i, result := range extractorResults { + builder.WriteString(result) + if i != len(extractorResults)-1 { + builder.WriteRune(',') + } + } + builder.WriteString("]") + } + builder.WriteRune('\n') + + return builder.String() +} diff --git a/pkg/executor/output_http.go b/pkg/executor/output_http.go new file mode 100644 index 000000000..d5cf2afa1 --- /dev/null +++ b/pkg/executor/output_http.go @@ -0,0 +1,41 @@ +package executor + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/pkg/matchers" + "github.com/projectdiscovery/retryablehttp-go" +) + +// buildOutputHTTP builds an output text for writing results +func (e *HTTPExecutor) buildOutputHTTP(req *retryablehttp.Request, extractorResults []string, matcher *matchers.Matcher) string { + builder := &strings.Builder{} + + builder.WriteRune('[') + builder.WriteString(e.template.ID) + if len(matcher.Name) > 0 { + builder.WriteString(":") + builder.WriteString(matcher.Name) + } + builder.WriteString("] [http] ") + + // Escape the URL by replacing all % with %% + URL := req.URL.String() + escapedURL := strings.Replace(URL, "%", "%%", -1) + builder.WriteString(escapedURL) + + // If any extractors, write the results + if len(extractorResults) > 0 { + builder.WriteString(" [") + for i, result := range extractorResults { + builder.WriteString(result) + if i != len(extractorResults)-1 { + builder.WriteRune(',') + } + } + builder.WriteString("]") + } + builder.WriteRune('\n') + + return builder.String() +} diff --git a/pkg/requests/dns-request.go b/pkg/requests/dns-request.go index f562a6ae6..46a021d48 100644 --- a/pkg/requests/dns-request.go +++ b/pkg/requests/dns-request.go @@ -1,7 +1,6 @@ package requests import ( - "fmt" "strings" "github.com/miekg/dns" @@ -29,7 +28,6 @@ type DNSRequest struct { // MakeDNSRequest creates a *dns.Request from a request template func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { - domain = dns.Fqdn(domain) // Build a request on the specified URL @@ -40,28 +38,16 @@ func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { var q dns.Question t := fasttemplate.New(r.Name, "{{", "}}") - q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{ - "FQDN": domain, - })) - - qclass, err := toQClass(r.Class) - if err != nil { - return nil, err - } - q.Qclass = qclass - - qtype, err := toQType(r.Type) - if err != nil { - return nil, err - } - q.Qtype = qtype + q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{"FQDN": domain})) + q.Qclass = toQClass(r.Class) + q.Qtype = toQType(r.Type) req.Question = append(req.Question, q) return req, nil } -func toQType(ttype string) (rtype uint16, err error) { +func toQType(ttype string) (rtype uint16) { ttype = strings.TrimSpace(strings.ToUpper(ttype)) switch ttype { @@ -82,14 +68,12 @@ func toQType(ttype string) (rtype uint16, err error) { case "AAAA": rtype = dns.TypeAAAA default: - rtype = dns.TypeNone - err = fmt.Errorf("incorrect type") + rtype = dns.TypeA } - return } -func toQClass(tclass string) (rclass uint16, err error) { +func toQClass(tclass string) (rclass uint16) { tclass = strings.TrimSpace(strings.ToUpper(tclass)) switch tclass { @@ -106,8 +90,8 @@ func toQClass(tclass string) (rclass uint16, err error) { case "ANY": rclass = dns.ClassANY default: - err = fmt.Errorf("incorrect class") + // Use INET by default. + rclass = dns.ClassINET } - return } diff --git a/pkg/requests/http-request.go b/pkg/requests/http-request.go index 7d831b240..01d5c622a 100644 --- a/pkg/requests/http-request.go +++ b/pkg/requests/http-request.go @@ -25,6 +25,9 @@ type HTTPRequest struct { // Matchers contains the detection mechanism for the request to identify // whether the request was successful Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` + // MatchersCondition is the condition of the matchers + // whether to use AND or OR. Default is OR. + MatchersCondition string `yaml:"matchers-condition,omitempty"` // Extractors contains the extraction mechanism for the request to identify // and extract parts of the response. Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 8c9352368..ac7e2d5b3 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -10,9 +10,9 @@ type Template struct { ID string `yaml:"id"` // Info contains information about the template Info Info `yaml:"info"` - // Request contains the http request to make in the template + // RequestHTTP contains the http request to make in the template RequestsHTTP []*requests.HTTPRequest `yaml:"requests"` - // Request contains the dns request to make in the template + // RequestDNS contains the dns request to make in the template RequestsDNS []*requests.DNSRequest `yaml:"dns"` }