From 041361c45ea13c4c7eaa8ab0826657d8cf926f2f Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 12 Nov 2021 19:29:45 +0100 Subject: [PATCH 01/41] Adding aws sign prototype --- v2/go.mod | 2 ++ v2/go.sum | 6 +++++ v2/pkg/protocols/http/build_request.go | 8 ++++++ v2/pkg/protocols/http/http.go | 1 + v2/pkg/protocols/http/request.go | 35 ++++++++++++++++++++++++++ v2/pkg/templates/compile.go | 5 ++++ v2/pkg/templates/templates.go | 1 + 7 files changed, 58 insertions(+) diff --git a/v2/go.mod b/v2/go.mod index 39a54ff3d..d7a1dfc67 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -77,6 +77,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aws/aws-sdk-go v1.42.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect @@ -99,6 +100,7 @@ require ( github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jasonlvhit/gocron v0.0.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect diff --git a/v2/go.sum b/v2/go.sum index a497178f8..47dfe6c05 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -100,6 +100,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.42.3 h1:lBKr3tQ06m1uykiychMNKLK1bRfOzaIEQpsI/S3QiNc= +github.com/aws/aws-sdk-go v1.42.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -392,6 +394,9 @@ github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrn github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -1206,6 +1211,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 4b4ac9f21..adf940af9 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -126,6 +127,13 @@ func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]int if len(parts) < 3 { return nil, fmt.Errorf("malformed request supplied") } + + // the url might contain placeholders + parts[1] = replacer.Replace(parts[1], generators.BuildPayloadFromOptions(r.request.options.Options)) + if expressions.ContainsUnresolvedVariables(parts[1]) != nil { + return nil, err + } + parsed, err := url.Parse(parts[1]) if err != nil { return nil, fmt.Errorf("could not parse request URL: %s", err) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 304f1a497..c1adef103 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -140,6 +140,7 @@ type Request struct { // description: | // SelfContained specifies if the request is self contained. SelfContained bool `yaml:"-" json:"-"` + AwsSign bool `yaml:"-" json:"-"` // description: | // CookieReuse is an optional setting that enables cookie reuse for diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3958d6c74..afb8efb8c 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/aws/aws-sdk-go/aws/credentials" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" @@ -342,6 +344,39 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } if resp == nil { + // aws sign the request if necessary + if request.AwsSign { + generatedRequest.request.Header.Del("Host") + payloads := request.options.Options.Vars.AsMap() + var creds *credentials.Credentials + if request.options.Options.EnvironmentVariables { + // get from env + creds = credentials.NewEnvCredentials() + } else { // get from variables { + awsAccessKeyId := payloads["aws-id"] + awsSecretAccessKey := payloads["aws-secret"] + creds = credentials.NewStaticCredentials(awsAccessKeyId.(string), awsSecretAccessKey.(string), "") + } + + signer := v4.NewSigner(creds) + var body *bytes.Reader + if generatedRequest.request.Request.Body != nil { + bodyBytes, err := ioutil.ReadAll(generatedRequest.request.Request.Body) + if err != nil { + return err + } + generatedRequest.request.Request.Body.Close() + body = bytes.NewReader(bodyBytes) + } + + service := payloads["service"].(string) + region := payloads["region"].(string) + now := time.Now() + _, err = signer.Sign(generatedRequest.request.Request, body, service, region, now) + if err != nil { + return err + } + } resp, err = request.httpClient.Do(generatedRequest.request) } } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index bf4e79b18..07ee2c753 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -152,6 +152,11 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // parseSelfContainedRequests parses the self contained template requests. func (t *Template) parseSelfContainedRequests() { + if t.AwsSign { + for _, request := range t.RequestsHTTP { + request.AwsSign = true + } + } if !t.SelfContained { return } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index ff9e527f4..f317e10bd 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -66,6 +66,7 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + AwsSign bool `yaml:"aws-sign,omitempty"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` From 34889d50f8c04465e37acb26a05522d62bf05703 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 12 Nov 2021 19:58:12 +0100 Subject: [PATCH 02/41] correcting variable name --- v2/go.sum | 1 + v2/pkg/templates/compile.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/go.sum b/v2/go.sum index 20ae01cc0..9ba1d4269 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -401,6 +401,7 @@ github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index abadb53dc..654abcb64 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -108,7 +108,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // parseSelfContainedRequests parses the self contained template requests. func (template *Template) parseSelfContainedRequests() { if template.AwsSign { - for _, request := range t.RequestsHTTP { + for _, request := range template.RequestsHTTP { request.AwsSign = true } } From e517797cfa6c767594ba4bae1c6282a19fc28393 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 13 Nov 2021 02:13:48 +0100 Subject: [PATCH 03/41] moving aws signing logic to helper library --- v2/pkg/protocols/http/aws_utils.go | 85 ++++++++++++++++++++++++++++++ v2/pkg/protocols/http/request.go | 39 ++++++-------- 2 files changed, 102 insertions(+), 22 deletions(-) create mode 100644 v2/pkg/protocols/http/aws_utils.go diff --git a/v2/pkg/protocols/http/aws_utils.go b/v2/pkg/protocols/http/aws_utils.go new file mode 100644 index 000000000..9209b3c33 --- /dev/null +++ b/v2/pkg/protocols/http/aws_utils.go @@ -0,0 +1,85 @@ +package http + +import ( + "bytes" + "context" + "errors" + "io/ioutil" + "net/http" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" + v4 "github.com/aws/aws-sdk-go/aws/signer/v4" +) + +type AwsSigner struct { + creds *credentials.Credentials + signer *v4.Signer +} + +type SignArguments struct { + Service string + Region string + Time time.Time +} + +func NewAwsSigner(awsId, awsSecretToken string) (*AwsSigner, error) { + if awsId == "" { + return nil, errors.New("empty id") + } + if awsSecretToken == "" { + return nil, errors.New("empty token") + } + + creds := credentials.NewStaticCredentials(awsId, awsSecretToken, "") + if creds == nil { + return nil, errors.New("couldn't create the credentials structure") + } + + signer := v4.NewSigner(creds) + + return &AwsSigner{creds: creds, signer: signer}, nil +} + +func NewAwsSignerFromEnv() (*AwsSigner, error) { + creds := credentials.NewEnvCredentials() + if creds == nil { + return nil, errors.New("couldn't create the credentials structure") + } + return &AwsSigner{creds: creds}, nil +} + +func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args SignArguments) error { + awsSigner.prepareRequest(request) + var body *bytes.Reader + if request.Body != nil { + bodyBytes, err := ioutil.ReadAll(request.Body) + if err != nil { + return err + } + request.Body.Close() + body = bytes.NewReader(bodyBytes) + } + + if _, err := awsSigner.signer.Sign(request, body, args.Service, args.Region, args.Time); err != nil { + return err + } + return nil +} + +func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args SignArguments) (map[string]string, error) { + reqClone := request.Clone(context.Background()) + awsSigner.prepareRequest(reqClone) + err := awsSigner.SignHTTP(reqClone, args) + if err != nil { + return nil, err + } + headers := make(map[string]string) + headers["X-Amz-Date"] = reqClone.Header.Get("X-Amz-Date") + headers["Authorization"] = reqClone.Header.Get("Authorization") + return headers, nil +} + +func (awsSigner *AwsSigner) prepareRequest(request *http.Request) { + request.Header.Del("Host") +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0a7c174db..eb8a716a3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -13,8 +13,6 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws/credentials" - v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" @@ -31,6 +29,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/stringsutil" ) @@ -352,33 +351,29 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if resp == nil { // aws sign the request if necessary if request.AwsSign { - generatedRequest.request.Header.Del("Host") + var awsSigner *AwsSigner payloads := request.options.Options.Vars.AsMap() - var creds *credentials.Credentials if request.options.Options.EnvironmentVariables { - // get from env - creds = credentials.NewEnvCredentials() - } else { // get from variables { - awsAccessKeyId := payloads["aws-id"] - awsSecretAccessKey := payloads["aws-secret"] - creds = credentials.NewStaticCredentials(awsAccessKeyId.(string), awsSecretAccessKey.(string), "") - } - - signer := v4.NewSigner(creds) - var body *bytes.Reader - if generatedRequest.request.Request.Body != nil { - bodyBytes, err := ioutil.ReadAll(generatedRequest.request.Request.Body) + var err error + awsSigner, err = NewAwsSignerFromEnv() + if err != nil { + return err + } + } else { // get from variables { + awsAccessKeyId := types.ToString(payloads["aws-id"]) + awsSecretAccessKey := types.ToString(payloads["aws-secret"]) + awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) if err != nil { return err } - generatedRequest.request.Request.Body.Close() - body = bytes.NewReader(bodyBytes) } - service := payloads["service"].(string) - region := payloads["region"].(string) - now := time.Now() - _, err = signer.Sign(generatedRequest.request.Request, body, service, region, now) + args := SignArguments{ + Service: types.ToString(payloads["service"]), + Region: types.ToString(payloads["region"]), + Time: time.Now(), + } + err = awsSigner.SignHTTP(generatedRequest.request.Request, args) if err != nil { return err } From eb61c519c3bfa38a324e4088a79b57de6d628c21 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 13 Nov 2021 03:17:05 +0100 Subject: [PATCH 04/41] converting to post-processor prototype --- v2/pkg/protocols/http/http.go | 4 +-- v2/pkg/protocols/http/request.go | 51 +++++++++++++++++++------------- v2/pkg/templates/compile.go | 4 +-- v2/pkg/templates/templates.go | 4 +-- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 7a6228b1b..765b217e6 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -138,8 +138,8 @@ type Request struct { // description: | // SelfContained specifies if the request is self contained. - SelfContained bool `yaml:"-" json:"-"` - AwsSign bool `yaml:"-" json:"-"` + SelfContained bool `yaml:"-" json:"-"` + PostProcessors []string `yaml:"-" json:"-"` // description: | // CookieReuse is an optional setting that enables cookie reuse for diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index eb8a716a3..6059f1679 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -197,6 +197,10 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous return requestErr } +func (request *Request) hasPreprocessors() bool { + return len(request.PostProcessors) > 0 +} + // ExecuteWithResults executes the final request on a URL func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested @@ -296,6 +300,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function if !generatedRequest.original.Race { var dumpError error + // TODO: dump is currently not working with post-processors - somehow it alters the signature dumpedRequest, dumpError = dump(generatedRequest, reqURL) if dumpError != nil { return dumpError @@ -349,33 +354,37 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } if resp == nil { - // aws sign the request if necessary - if request.AwsSign { - var awsSigner *AwsSigner - payloads := request.options.Options.Vars.AsMap() - if request.options.Options.EnvironmentVariables { - var err error - awsSigner, err = NewAwsSignerFromEnv() - if err != nil { - return err - } - } else { // get from variables { + for _, postProcessor := range request.PostProcessors { + switch postProcessor { + case "aws-sign": + var awsSigner *AwsSigner + payloads := request.options.Options.Vars.AsMap() awsAccessKeyId := types.ToString(payloads["aws-id"]) awsSecretAccessKey := types.ToString(payloads["aws-secret"]) - awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) + } else { + awsSigner, err = NewAwsSignerFromEnv() + } if err != nil { return err } - } - args := SignArguments{ - Service: types.ToString(payloads["service"]), - Region: types.ToString(payloads["region"]), - Time: time.Now(), - } - err = awsSigner.SignHTTP(generatedRequest.request.Request, args) - if err != nil { - return err + service := types.ToString(payloads["service"]) + region := types.ToString(payloads["region"]) + if service == "" || region == "" { + return errors.New("service and region are mandatory") + } + + args := SignArguments{ + Service: types.ToString(payloads["service"]), + Region: types.ToString(payloads["region"]), + Time: time.Now(), + } + err = awsSigner.SignHTTP(generatedRequest.request.Request, args) + if err != nil { + return err + } } } resp, err = request.httpClient.Do(generatedRequest.request) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 654abcb64..97f1afa70 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -107,9 +107,9 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // parseSelfContainedRequests parses the self contained template requests. func (template *Template) parseSelfContainedRequests() { - if template.AwsSign { + if len(template.PostProcessors) > 0 { for _, request := range template.RequestsHTTP { - request.AwsSign = true + request.PostProcessors = template.PostProcessors } } if !template.SelfContained { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b4648d990..7d5ae7702 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,8 +73,8 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained - SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` - AwsSign bool `yaml:"aws-sign,omitempty"` + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + PostProcessors []string `yaml:"post-processors,omitempty"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` From 38ff8f75b1377424c5caa37fb06157c1b0a33d1c Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 17 Nov 2021 01:28:35 +0100 Subject: [PATCH 05/41] adding signature syntax validation --- v2/pkg/protocols/http/http.go | 9 ++- v2/pkg/protocols/http/request.go | 60 +++++++++----------- v2/pkg/protocols/http/signature.go | 89 ++++++++++++++++++++++++++++++ v2/pkg/templates/compile.go | 4 +- v2/pkg/templates/templates.go | 9 ++- 5 files changed, 132 insertions(+), 39 deletions(-) create mode 100644 v2/pkg/protocols/http/signature.go diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 765b217e6..ed6b9b2a6 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -138,8 +138,13 @@ type Request struct { // description: | // SelfContained specifies if the request is self contained. - SelfContained bool `yaml:"-" json:"-"` - PostProcessors []string `yaml:"-" json:"-"` + SelfContained bool `yaml:"-" json:"-"` + + // description: | + // Signature is the request signature method + // values: + // - "AWS" + Signature SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` // description: | // CookieReuse is an optional setting that enables cookie reuse for diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 6059f1679..9849a5c01 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -197,10 +197,6 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous return requestErr } -func (request *Request) hasPreprocessors() bool { - return len(request.PostProcessors) > 0 -} - // ExecuteWithResults executes the final request on a URL func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested @@ -354,37 +350,35 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } if resp == nil { - for _, postProcessor := range request.PostProcessors { - switch postProcessor { - case "aws-sign": - var awsSigner *AwsSigner - payloads := request.options.Options.Vars.AsMap() - awsAccessKeyId := types.ToString(payloads["aws-id"]) - awsSecretAccessKey := types.ToString(payloads["aws-secret"]) - if awsAccessKeyId != "" && awsSecretAccessKey != "" { - awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) - } else { - awsSigner, err = NewAwsSignerFromEnv() - } - if err != nil { - return err - } + switch request.Signature.Value { + case AWSSignature: + var awsSigner *AwsSigner + payloads := request.options.Options.Vars.AsMap() + awsAccessKeyId := types.ToString(payloads["aws-id"]) + awsSecretAccessKey := types.ToString(payloads["aws-secret"]) + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) + } else { + awsSigner, err = NewAwsSignerFromEnv() + } + if err != nil { + return err + } - service := types.ToString(payloads["service"]) - region := types.ToString(payloads["region"]) - if service == "" || region == "" { - return errors.New("service and region are mandatory") - } + service := types.ToString(payloads["service"]) + region := types.ToString(payloads["region"]) + if service == "" || region == "" { + return errors.New("service and region are mandatory") + } - args := SignArguments{ - Service: types.ToString(payloads["service"]), - Region: types.ToString(payloads["region"]), - Time: time.Now(), - } - err = awsSigner.SignHTTP(generatedRequest.request.Request, args) - if err != nil { - return err - } + args := SignArguments{ + Service: types.ToString(payloads["service"]), + Region: types.ToString(payloads["region"]), + Time: time.Now(), + } + err = awsSigner.SignHTTP(generatedRequest.request.Request, args) + if err != nil { + return err } } resp, err = request.httpClient.Do(generatedRequest.request) diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go new file mode 100644 index 000000000..83f4a7d5e --- /dev/null +++ b/v2/pkg/protocols/http/signature.go @@ -0,0 +1,89 @@ +package http + +import ( + "encoding/json" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" +) + +// SignatureType is the type of signature +type SignatureType int + +// Supported values for the SignatureType +const ( + AWSSignature SignatureType = iota + 1 + limit +) + +// signatureTypeMappings is a table for conversion of signature type from string. +var signatureTypeMappings = map[SignatureType]string{ + AWSSignature: "aws", +} + +func GetSupportedSignaturesTypes() []SignatureType { + var result []SignatureType + for index := SignatureType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toSignatureType(valueToMap string) (SignatureType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range signatureTypeMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("invalid signature type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t SignatureType) String() string { + return signatureTypeMappings[t] +} + +// SignatureTypeHolder is used to hold internal type of the signature +type SignatureTypeHolder struct { + Value SignatureType +} + +func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the signature", + Description: "Type of the signature", + } + for _, types := range GetSupportedSignaturesTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toSignatureType(marshalledTypes) + if err != nil { + return err + } + + holder.Value = computedType + return nil +} + +func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.Value.String()) +} + +func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) { + return holder.Value.String(), nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 97f1afa70..092ec1d3e 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -107,9 +107,9 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute // parseSelfContainedRequests parses the self contained template requests. func (template *Template) parseSelfContainedRequests() { - if len(template.PostProcessors) > 0 { + if template.Signature.Value.String() != "" { for _, request := range template.RequestsHTTP { - request.PostProcessors = template.PostProcessors + request.Signature = template.Signature } } if !template.SelfContained { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index ed74175d7..857c8823a 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,8 +73,13 @@ type Template struct { // description: | // Self Contained marks Requests for the template as self-contained - SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` - PostProcessors []string `yaml:"post-processors,omitempty"` + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + + // description: | + // Signature is the request signature method + // values: + // - "AWS" + Signature http.SignatureTypeHolder `yaml:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` From ddb07ebb75692d6f8c4535101282c7543ad44469 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 18 Nov 2021 21:44:58 +0100 Subject: [PATCH 06/41] adding support for aws file credential --- v2/pkg/protocols/http/aws_utils.go | 8 ++++++++ v2/pkg/protocols/http/request.go | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/v2/pkg/protocols/http/aws_utils.go b/v2/pkg/protocols/http/aws_utils.go index 9209b3c33..c5663c20d 100644 --- a/v2/pkg/protocols/http/aws_utils.go +++ b/v2/pkg/protocols/http/aws_utils.go @@ -49,6 +49,14 @@ func NewAwsSignerFromEnv() (*AwsSigner, error) { return &AwsSigner{creds: creds}, nil } +func NewAwsSignerFromFile() (*AwsSigner, error) { + creds := credentials.NewSharedCredentials("", "") + if creds == nil { + return nil, errors.New("couldn't create the credentials structure") + } + return &AwsSigner{creds: creds}, nil +} + func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args SignArguments) error { awsSigner.prepareRequest(request) var body *bytes.Reader diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 9849a5c01..d97f65140 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -359,7 +359,15 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if awsAccessKeyId != "" && awsSecretAccessKey != "" { awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) } else { + // env variables awsSigner, err = NewAwsSignerFromEnv() + if err != nil { + // $HOME/.aws/credentials + awsSigner, err = NewAwsSignerFromFile() + if err != nil { + return err + } + } } if err != nil { return err From bdffa373cff4fa764d04e70860773680e9a1db7e Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 18 Nov 2021 21:54:24 +0100 Subject: [PATCH 07/41] solving linting issues --- v2/pkg/protocols/http/signature.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index 83f4a7d5e..cf01f1e73 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -2,7 +2,6 @@ package http import ( "encoding/json" - "strings" "github.com/alecthomas/jsonschema" "github.com/pkg/errors" @@ -14,7 +13,7 @@ type SignatureType int // Supported values for the SignatureType const ( AWSSignature SignatureType = iota + 1 - limit + signatureLimit ) // signatureTypeMappings is a table for conversion of signature type from string. @@ -24,7 +23,7 @@ var signatureTypeMappings = map[SignatureType]string{ func GetSupportedSignaturesTypes() []SignatureType { var result []SignatureType - for index := SignatureType(1); index < limit; index++ { + for index := SignatureType(1); index < signatureLimit; index++ { result = append(result, index) } return result @@ -40,10 +39,6 @@ func toSignatureType(valueToMap string) (SignatureType, error) { return -1, errors.New("invalid signature type: " + valueToMap) } -func normalizeValue(value string) string { - return strings.TrimSpace(strings.ToLower(value)) -} - func (t SignatureType) String() string { return signatureTypeMappings[t] } From 359eb7c3d287bdc7c6b6817e625c79ba38fe7e8f Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 18 Nov 2021 21:58:32 +0100 Subject: [PATCH 08/41] making iota zero based --- v2/pkg/protocols/http/signature.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index cf01f1e73..a8b4c7bd6 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -12,8 +12,7 @@ type SignatureType int // Supported values for the SignatureType const ( - AWSSignature SignatureType = iota + 1 - signatureLimit + AWSSignature SignatureType = iota ) // signatureTypeMappings is a table for conversion of signature type from string. @@ -23,8 +22,8 @@ var signatureTypeMappings = map[SignatureType]string{ func GetSupportedSignaturesTypes() []SignatureType { var result []SignatureType - for index := SignatureType(1); index < signatureLimit; index++ { - result = append(result, index) + for i := 0; SignatureType(i).String() != ""; i++ { + result = append(result, SignatureType(i)) } return result } From 71d4e5318a96d605e1caf07761329862849181da Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 18 Nov 2021 22:28:10 +0100 Subject: [PATCH 09/41] . --- v2/pkg/protocols/http/signature.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index a8b4c7bd6..cf01f1e73 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -12,7 +12,8 @@ type SignatureType int // Supported values for the SignatureType const ( - AWSSignature SignatureType = iota + AWSSignature SignatureType = iota + 1 + signatureLimit ) // signatureTypeMappings is a table for conversion of signature type from string. @@ -22,8 +23,8 @@ var signatureTypeMappings = map[SignatureType]string{ func GetSupportedSignaturesTypes() []SignatureType { var result []SignatureType - for i := 0; SignatureType(i).String() != ""; i++ { - result = append(result, SignatureType(i)) + for index := SignatureType(1); index < signatureLimit; index++ { + result = append(result, index) } return result } From ed55de71d914fd3181799042aee8df3e6c74784b Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 20 Nov 2021 02:26:16 +0100 Subject: [PATCH 10/41] fixing internal normalized value --- v2/pkg/protocols/http/signature.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index cf01f1e73..53f31164f 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -18,7 +18,7 @@ const ( // signatureTypeMappings is a table for conversion of signature type from string. var signatureTypeMappings = map[SignatureType]string{ - AWSSignature: "aws", + AWSSignature: "AWS", } func GetSupportedSignaturesTypes() []SignatureType { From a3319930c02d4aea7a1ed69fd582c168479641e4 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 26 Nov 2021 13:49:12 +0100 Subject: [PATCH 11/41] code refactor + request dump fix Fixes indirectly #844 --- v2/pkg/protocols/common/protocolinit/init.go | 4 ++ v2/pkg/protocols/http/request.go | 60 +++++++++-------- .../http/{aws_utils.go => signer/aws.go} | 64 +++++++++++++++---- v2/pkg/protocols/http/signer/signer.go | 40 ++++++++++++ .../protocols/http/signerpool/signerpool.go | 57 +++++++++++++++++ 5 files changed, 182 insertions(+), 43 deletions(-) rename v2/pkg/protocols/http/{aws_utils.go => signer/aws.go} (53%) create mode 100644 v2/pkg/protocols/http/signer/signer.go create mode 100644 v2/pkg/protocols/http/signerpool/signerpool.go diff --git a/v2/pkg/protocols/common/protocolinit/init.go b/v2/pkg/protocols/common/protocolinit/init.go index 1877a1ab3..7b257f707 100644 --- a/v2/pkg/protocols/common/protocolinit/init.go +++ b/v2/pkg/protocols/common/protocolinit/init.go @@ -5,6 +5,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -22,6 +23,9 @@ func Init(options *types.Options) error { if err := httpclientpool.Init(options); err != nil { return err } + if err := signerpool.Init(options); err != nil { + return err + } return networkclientpool.Init(options) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d97f65140..59867cdd1 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -28,6 +28,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" @@ -293,6 +295,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate err error ) + // Dump request for variables checks // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function if !generatedRequest.original.Race { var dumpError error @@ -308,11 +311,6 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) return errStopExecution } - - if request.options.Options.Debug || request.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) - gologger.Print().Msgf("%s", dumpedRequestString) - } } var formedURL string @@ -352,39 +350,24 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if resp == nil { switch request.Signature.Value { case AWSSignature: - var awsSigner *AwsSigner + var awsSigner signer.Signer payloads := request.options.Options.Vars.AsMap() awsAccessKeyId := types.ToString(payloads["aws-id"]) awsSecretAccessKey := types.ToString(payloads["aws-secret"]) - if awsAccessKeyId != "" && awsSecretAccessKey != "" { - awsSigner, err = NewAwsSigner(awsAccessKeyId, awsSecretAccessKey) - } else { - // env variables - awsSigner, err = NewAwsSignerFromEnv() - if err != nil { - // $HOME/.aws/credentials - awsSigner, err = NewAwsSignerFromFile() - if err != nil { - return err - } - } + awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey} + service := types.ToString(payloads["service"]) + region := types.ToString(payloads["region"]) + awsSignatureArguments := signer.AwsSignatureArguments{ + Service: types.ToString(service), + Region: types.ToString(region), + Time: time.Now(), } + + awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs}) if err != nil { return err } - - service := types.ToString(payloads["service"]) - region := types.ToString(payloads["region"]) - if service == "" || region == "" { - return errors.New("service and region are mandatory") - } - - args := SignArguments{ - Service: types.ToString(payloads["service"]), - Region: types.ToString(payloads["region"]), - Time: time.Now(), - } - err = awsSigner.SignHTTP(generatedRequest.request.Request, args) + err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments) if err != nil { return err } @@ -392,6 +375,21 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate resp, err = request.httpClient.Do(generatedRequest.request) } } + + // Dump the requests containing all headers + if !generatedRequest.original.Race { + var dumpError error + dumpedRequest, dumpError = dump(generatedRequest, reqURL) + if dumpError != nil { + return dumpError + } + dumpedRequestString := string(dumpedRequest) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) + gologger.Print().Msgf("%s", dumpedRequestString) + } + } + if err != nil { // rawhttp doesn't support draining response bodies. if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil { diff --git a/v2/pkg/protocols/http/aws_utils.go b/v2/pkg/protocols/http/signer/aws.go similarity index 53% rename from v2/pkg/protocols/http/aws_utils.go rename to v2/pkg/protocols/http/signer/aws.go index c5663c20d..ccd852c8d 100644 --- a/v2/pkg/protocols/http/aws_utils.go +++ b/v2/pkg/protocols/http/signer/aws.go @@ -1,4 +1,4 @@ -package http +package signer import ( "bytes" @@ -17,21 +17,44 @@ type AwsSigner struct { signer *v4.Signer } -type SignArguments struct { +type AwsSignerArgs struct { + AwsId string + AwsSecretToken string +} + +func (awsSignerArgs AwsSignerArgs) Validate() error { + if awsSignerArgs.AwsId == "" { + return errors.New("empty id") + } + if awsSignerArgs.AwsSecretToken == "" { + return errors.New("empty token") + } + + return nil +} + +type AwsSignatureArguments struct { Service string Region string Time time.Time } -func NewAwsSigner(awsId, awsSecretToken string) (*AwsSigner, error) { - if awsId == "" { - return nil, errors.New("empty id") +func (awsSignatureArguments AwsSignatureArguments) Validate() error { + if awsSignatureArguments.Region == "" { + return errors.New("empty region") } - if awsSecretToken == "" { - return nil, errors.New("empty token") + if awsSignatureArguments.Service == "" { + return errors.New("empty service") } - creds := credentials.NewStaticCredentials(awsId, awsSecretToken, "") + return nil +} + +func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) { + if err := args.Validate(); err != nil { + return nil, err + } + creds := credentials.NewStaticCredentials(args.AwsId, args.AwsSecretToken, "") if creds == nil { return nil, errors.New("couldn't create the credentials structure") } @@ -57,7 +80,12 @@ func NewAwsSignerFromFile() (*AwsSigner, error) { return &AwsSigner{creds: creds}, nil } -func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args SignArguments) error { +func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error { + signatureArgs, err := awsSigner.checkSignatureArgs(args) + if err != nil { + return err + } + awsSigner.prepareRequest(request) var body *bytes.Reader if request.Body != nil { @@ -69,16 +97,21 @@ func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args SignArguments) body = bytes.NewReader(bodyBytes) } - if _, err := awsSigner.signer.Sign(request, body, args.Service, args.Region, args.Time); err != nil { + if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil { return err } return nil } -func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args SignArguments) (map[string]string, error) { +func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error) { + signatureArgs, err := awsSigner.checkSignatureArgs(args) + if err != nil { + return nil, err + } + reqClone := request.Clone(context.Background()) awsSigner.prepareRequest(reqClone) - err := awsSigner.SignHTTP(reqClone, args) + err = awsSigner.SignHTTP(reqClone, signatureArgs) if err != nil { return nil, err } @@ -88,6 +121,13 @@ func (awsSigner *AwsSigner) CalculateHTTPHeaders(request *http.Request, args Sig return headers, nil } +func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureArguments, error) { + if signatureArgs, ok := args.(AwsSignatureArguments); ok { + return signatureArgs, signatureArgs.Validate() + } + return AwsSignatureArguments{}, errors.New("wrong signature type") +} + func (awsSigner *AwsSigner) prepareRequest(request *http.Request) { request.Header.Del("Host") } diff --git a/v2/pkg/protocols/http/signer/signer.go b/v2/pkg/protocols/http/signer/signer.go new file mode 100644 index 000000000..262d17bb6 --- /dev/null +++ b/v2/pkg/protocols/http/signer/signer.go @@ -0,0 +1,40 @@ +package signer + +import ( + "errors" + "net/http" +) + +type Signer interface { + SignHTTP(request *http.Request, args interface{}) error + CalculateHTTPHeaders(request *http.Request, args interface{}) (map[string]string, error) +} + +type SignerArgs interface { + Validate() error +} + +type SignatureArguments interface { + Validate() error +} + +func NewSigner(args SignerArgs) (signer Signer, err error) { + switch signerArgs := args.(type) { + case AwsSignerArgs: + awsSigner, err := NewAwsSigner(signerArgs) + if err != nil { + // env variables + awsSigner, err = NewAwsSignerFromEnv() + if err != nil { + // $HOME/.aws/credentials + awsSigner, err = NewAwsSignerFromFile() + if err != nil { + return nil, err + } + } + } + return awsSigner, err + default: + return nil, errors.New("unknown signature arguments type") + } +} diff --git a/v2/pkg/protocols/http/signerpool/signerpool.go b/v2/pkg/protocols/http/signerpool/signerpool.go new file mode 100644 index 000000000..94fd9a7ca --- /dev/null +++ b/v2/pkg/protocols/http/signerpool/signerpool.go @@ -0,0 +1,57 @@ +package signerpool + +import ( + "fmt" + "strings" + "sync" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var ( + poolMutex *sync.RWMutex + clientPool map[string]signer.Signer +) + +// Init initializes the clientpool implementation +func Init(options *types.Options) error { + poolMutex = &sync.RWMutex{} + clientPool = make(map[string]signer.Signer) + return nil +} + +// Configuration contains the custom configuration options for a client +type Configuration struct { + SignerArgs signer.SignerArgs +} + +// Hash returns the hash of the configuration to allow client pooling +func (c *Configuration) Hash() string { + builder := &strings.Builder{} + builder.WriteString(fmt.Sprintf("%v", c.SignerArgs)) + hash := builder.String() + return hash +} + +// Get creates or gets a client for the protocol based on custom configuration +func Get(options *types.Options, configuration *Configuration) (signer.Signer, error) { + hash := configuration.Hash() + poolMutex.RLock() + if client, ok := clientPool[hash]; ok { + poolMutex.RUnlock() + return client, nil + } + poolMutex.RUnlock() + + client, err := signer.NewSigner(configuration.SignerArgs) + if err != nil { + return nil, err + } + + poolMutex.Lock() + clientPool[hash] = client + poolMutex.Unlock() + return client, nil +} From c9b9725474b764deff71b3591524a50f82ae2c68 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 2 Dec 2021 15:57:52 +0100 Subject: [PATCH 12/41] moving sign routine to helper function --- v2/pkg/protocols/http/request.go | 55 +++++++++++++++++++------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 599e6c11d..41c11077b 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -391,29 +391,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } } if resp == nil { - switch request.Signature.Value { - case AWSSignature: - var awsSigner signer.Signer - payloads := request.options.Options.Vars.AsMap() - awsAccessKeyId := types.ToString(payloads["aws-id"]) - awsSecretAccessKey := types.ToString(payloads["aws-secret"]) - awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey} - service := types.ToString(payloads["service"]) - region := types.ToString(payloads["region"]) - awsSignatureArguments := signer.AwsSignatureArguments{ - Service: types.ToString(service), - Region: types.ToString(region), - Time: time.Now(), - } - - awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs}) - if err != nil { - return err - } - err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments) - if err != nil { - return err - } + if errSignature := request.handleSignature(generatedRequest); errSignature != nil { + return errSignature } resp, err = request.httpClient.Do(generatedRequest.request) } @@ -572,6 +551,36 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate return nil } +// handleSignature of the http request +func (request *Request) handleSignature(generatedRequest *generatedRequest) error { + switch request.Signature.Value { + case AWSSignature: + var awsSigner signer.Signer + payloads := request.options.Options.Vars.AsMap() + awsAccessKeyId := types.ToString(payloads["aws-id"]) + awsSecretAccessKey := types.ToString(payloads["aws-secret"]) + awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey} + service := types.ToString(payloads["service"]) + region := types.ToString(payloads["region"]) + awsSignatureArguments := signer.AwsSignatureArguments{ + Service: types.ToString(service), + Region: types.ToString(region), + Time: time.Now(), + } + + awsSigner, err := signerpool.Get(request.options.Options, &signerpool.Configuration{SignerArgs: awsSignerArgs}) + if err != nil { + return err + } + err = awsSigner.SignHTTP(generatedRequest.request.Request, awsSignatureArguments) + if err != nil { + return err + } + } + + return nil +} + // setCustomHeaders sets the custom headers for generated request func (request *Request) setCustomHeaders(req *generatedRequest) { for k, v := range request.customHeaders { From aeab730dc2c88b9576de1f1e5568f0f823cd1955 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 7 Dec 2021 09:59:17 +0100 Subject: [PATCH 13/41] fixing merge conflicts --- v2/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/go.sum b/v2/go.sum index ddf868b32..7256aaa67 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -601,8 +601,6 @@ github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/folderutil v0.0.0-20211206102047-d6bf8e7490ff h1:ci7/Pq9xvrVFb94jeARYb45oSzs85NWG+Fxp/kjgHVc= -github.com/projectdiscovery/folderutil v0.0.0-20211206102047-d6bf8e7490ff/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e h1:RJJuYyuwskYtzZi2gziy6SE/b7saWEzyskaA252E0VY= github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= From abb78658c69c38741b36dfc6a2c6e7109070ba89 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 9 Dec 2021 08:50:54 +0100 Subject: [PATCH 14/41] adding default region --- v2/pkg/protocols/http/request.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index d416dfb34..11902d67d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -562,6 +562,10 @@ func (request *Request) handleSignature(generatedRequest *generatedRequest) erro awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey} service := types.ToString(payloads["service"]) region := types.ToString(payloads["region"]) + // if region is empty default to "us-east-2" + if region == "" { + region = "us-east-2" + } awsSignatureArguments := signer.AwsSignatureArguments{ Service: types.ToString(service), Region: types.ToString(region), From db530c851a7ac8979bb3547e27c752c0a98ce96f Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 10 Dec 2021 21:47:47 +0530 Subject: [PATCH 15/41] fix: docker build fix --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index cc6775e82..b849631e0 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -35,6 +35,6 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64,linux/arm64,linux/arm + platforms: linux/amd64,linux/arm64 push: true tags: projectdiscovery/nuclei:latest,projectdiscovery/nuclei:${{ steps.meta.outputs.tag }} \ No newline at end of file From ed309e446ab6284b74b7e4a58091e4628d140251 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Mon, 13 Dec 2021 18:46:41 +0530 Subject: [PATCH 16/41] Interactsh bugfix release (#1368) * dev version update * Adding race condition test (#1357) * Removing linux/arm - Missing chrome package (#1355) * Trim paths info from released binary (#1356) * chore(deps): bump golang from 1.17.4-alpine to 1.17.5-alpine (#1363) Bumps golang from 1.17.4-alpine to 1.17.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix: updating default interactsh server to use Co-authored-by: Mzack9999 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-test.yml | 5 +++++ Dockerfile | 2 +- README.md | 2 +- README_CN.md | 2 +- v2/.goreleaser.yml | 3 +++ v2/cmd/nuclei/main.go | 2 +- v2/pkg/catalog/config/config.go | 2 +- v2/pkg/protocols/common/interactsh/interactsh.go | 4 ++-- v2/pkg/testutils/testutils.go | 2 +- 9 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 52751ee19..9e1935041 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -28,4 +28,9 @@ jobs: - name: Build run: go build . + working-directory: v2/cmd/nuclei/ + + # At the bottom for known issue on OSX - https://github.com/projectdiscovery/nuclei/issues/1309 + - name: Race Condition Tests + run: go build -race . working-directory: v2/cmd/nuclei/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 21822611b..a0806c9f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.4-alpine as build-env +FROM golang:1.17.5-alpine as build-env RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.15.0 diff --git a/README.md b/README.md index 1b3dc3adf..83a2e24f0 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ CONFIGURATIONS: -ca, -client-ca string client certificate authority file (PEM-encoded) used for authenticating against scanned hosts INTERACTSH: - -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") + -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interact.sh") -itoken, -interactsh-token string authentication token for self-hosted interactsh server -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) diff --git a/README_CN.md b/README_CN.md index b52263563..74f401692 100644 --- a/README_CN.md +++ b/README_CN.md @@ -133,7 +133,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板 -ev, env-vars 在模板中使用环境变量 交互： - -inserver, -ineractsh-server string 使用interactsh反连检测平台（默认为"https://interactsh.com"） + -inserver, -ineractsh-server string 使用interactsh反连检测平台（默认为"https://interact.sh"） -itoken, -interactsh-token string 指定反连检测平台的身份凭证 -interactions-cache-size int 指定保存在交互缓存中的请求数（默认：5000） -interactions-eviction int 聪缓存中删除请求前等待的时间（默认为60秒） diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 8ae96e831..ba272fcf3 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -26,6 +26,9 @@ builds: binary: '{{ .ProjectName }}' main: cmd/nuclei/main.go + flags: + - -trimpath + archives: - format: zip replacements: diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index bd015fede..cdaa3f39a 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -105,7 +105,7 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "interactsh", "interactsh", - flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interactsh.com", "interactsh server url for self-hosted instance"), + flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interact.sh", "interactsh server url for self-hosted instance"), flagSet.StringVarP(&options.InteractshToken, "interactsh-token", "itoken", "", "authentication token for self-hosted interactsh server"), flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index bb5cfb24f..c01bf4cd3 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.4` +const Version = `2.5.5` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index b810c346d..07dc19de6 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -109,7 +109,7 @@ func New(options *Options) (*Client, error) { // NewDefaultOptions returns the default options for interactsh client func NewDefaultOptions(output output.Writer, reporting *reporting.Client, progress progress.Progress) *Options { return &Options{ - ServerURL: "https://interactsh.com", + ServerURL: "https://interact.sh", CacheSize: 5000, Eviction: 60 * time.Second, ColldownPeriod: 5 * time.Second, @@ -265,7 +265,7 @@ func (c *Client) RequestEvent(interactshURLs []string, data *RequestData) { // HasMatchers returns true if an operator has interactsh part // matchers or extractors. // -// Used by requests to show result or not depending on presence of interactsh.com +// Used by requests to show result or not depending on presence of interact.sh // data part matchers. func HasMatchers(op *operators.Operators) bool { if op == nil { diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 99ddf9cef..b39752290 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -58,7 +58,7 @@ var DefaultOptions = &types.Options{ Templates: []string{}, ExcludedTemplates: []string{}, CustomHeaders: []string{}, - InteractshURL: "https://interactsh.com", + InteractshURL: "https://interact.sh", InteractionsCacheSize: 5000, InteractionsEviction: 60, InteractionsCoolDownPeriod: 5, From 3f86bc7d621d95e33777094caf6b82d68d811360 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 16 Dec 2021 02:17:29 -0600 Subject: [PATCH 17/41] Support major os (#1347) * workflow check * changes in run.sh file for windows support --- .github/workflows/build-test.yml | 5 ++++- .github/workflows/functional-test.yml | 9 ++++++--- v2/cmd/functional-test/run.sh | 17 ++++++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) mode change 100644 => 100755 v2/cmd/functional-test/run.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e0eb80594..2de3eec15 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -7,7 +7,10 @@ on: jobs: build: name: Test Builds - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest, macos-11.0] steps: - name: Set up Go uses: actions/setup-go@v2 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 902a662dc..9ea360751 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -8,7 +8,10 @@ on: jobs: functional: name: Functional Test - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest, macos-11.0] steps: - name: Set up Go uses: actions/setup-go@v2 @@ -23,5 +26,5 @@ jobs: GH_ACTION: true run: | chmod +x run.sh - bash run.sh - working-directory: v2/cmd/functional-test \ No newline at end of file + bash run.sh ${{ matrix.os }} + working-directory: v2/cmd/functional-test diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh old mode 100644 new mode 100755 index d37d29061..f53978219 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -1,16 +1,23 @@ #!/bin/bash +# reading os type from arguments +CURRENT_OS=$1 + +if [ "${CURRENT_OS}" == "windows-latest" ];then + extension=.exe +fi + echo "::group::Building functional-test binary" -go build +go build -o functional-test$extension echo "::endgroup::" echo "::group::Building Nuclei binary from current branch" -go build -o nuclei_dev ../nuclei +go build -o nuclei_dev$extension ../nuclei echo "::endgroup::" -echo "::group::Installing latest release of nuclei" -GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +echo "::group::Building latest release of nuclei" +go build -o nuclei$extension -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo "::endgroup::" echo 'Starting Nuclei functional test' -./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file +./functional-test$extension -main ./nuclei$extension -dev ./nuclei_dev$extension -testcases testcases.txt From fea3fabdf2c0136fb638d45335b103dadf00d78b Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 16 Dec 2021 13:59:19 +0530 Subject: [PATCH 18/41] Misc changes to update logic (#1212) * Misc changes to update logic * Misc adjustments to update logic * update: build check * update: revert test update Co-authored-by: sandeep --- v2/internal/runner/options.go | 4 ++++ v2/internal/runner/runner.go | 2 +- v2/internal/runner/update.go | 12 ++++-------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 54a79ac8d..9fee6c813 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -29,6 +29,10 @@ func ParseOptions(options *types.Options) { // Show the user the banner showBanner() + if !filepath.IsAbs(options.TemplatesDirectory) { + cwd, _ := os.Getwd() + options.TemplatesDirectory = filepath.Join(cwd, options.TemplatesDirectory) + } if options.Version { gologger.Info().Msgf("Current Version: %s\n", config.Version) os.Exit(0) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b71587267..56eb05360 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -73,7 +73,7 @@ func New(options *types.Options) (*Runner, error) { options.NoUpdateTemplates = true } if err := runner.updateTemplates(); err != nil { - gologger.Warning().Msgf("Could not update templates: %s\n", err) + gologger.Error().Msgf("Could not update templates: %s\n", err) } if options.Headless { if engine.MustDisableSandbox() { diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 1cf4ccf5b..bc19933a6 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -119,7 +119,7 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju if err := config.WriteConfiguration(r.templatesConfig); err != nil { return err } - gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s). GoodLuck!\n", version.String()) + gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory) return nil } @@ -135,13 +135,13 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju return config.WriteConfiguration(r.templatesConfig) } - if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil { + if err := r.updateTemplatesWithVersion(latestVersion, currentVersion, r, ctx); err != nil { return err } return nil } -func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { +func (r *Runner) updateTemplatesWithVersion(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { if latestVersion.GT(currentVersion) { gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) gologger.Info().Msgf("Downloading latest release...") @@ -163,7 +163,7 @@ func updateTemplates(latestVersion semver.Version, currentVersion semver.Version if err := config.WriteConfiguration(runner.templatesConfig); err != nil { return err } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String()) + gologger.Info().Msgf("Successfully updated nuclei-templates (v%s) to %s. GoodLuck!\n", latestVersion.String(), r.templatesConfig.TemplatesDirectory) } return nil } @@ -200,10 +200,6 @@ func (r *Runner) readInternalConfigurationFile(home, configDir string) error { return readErr } r.templatesConfig = configuration - - if configuration.TemplatesDirectory != "" && configuration.TemplatesDirectory != filepath.Join(home, "nuclei-templates") { - r.options.TemplatesDirectory = configuration.TemplatesDirectory - } } return nil } From 1fbbce4e41c298efe5b5dbb356f29faa5ae7e237 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Dec 2021 11:09:38 +0100 Subject: [PATCH 19/41] Adding support for implicit validation during marshal/unmarshal (#1329) --- v2/go.mod | 7 ++++-- v2/pkg/templates/templates.go | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 38a1594d3..41dbc4d87 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,7 +3,6 @@ module github.com/projectdiscovery/nuclei/v2 go 1.17 require ( - github.com/Ice3man543/nvd v1.0.8 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.14.0 @@ -47,7 +46,6 @@ require ( github.com/shirou/gopsutil/v3 v3.21.9 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.4.1 - github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/valyala/fasttemplate v1.2.1 @@ -67,6 +65,11 @@ require ( require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36e +require ( + github.com/Ice3man543/nvd v1.0.8 + github.com/stretchr/testify v1.7.0 +) + require ( git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect github.com/PuerkitoBio/goquery v1.6.0 // indirect diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 0eb400afe..bdf595cf0 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -2,6 +2,9 @@ package templates import ( + "encoding/json" + + validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" @@ -13,6 +16,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" + "go.uber.org/multierr" + "gopkg.in/yaml.v2" ) // Template is a YAML input file which defines all the requests and @@ -121,3 +126,41 @@ func (template *Template) Type() types.ProtocolType { return types.InvalidProtocol } } + +// MarshalYAML forces recursive struct validation during marshal operation +func (template *Template) MarshalYAML() ([]byte, error) { + out, marshalErr := yaml.Marshal(template) + errValidate := validate.New().Struct(template) + return out, multierr.Append(marshalErr, errValidate) +} + +// MarshalYAML forces recursive struct validation after unmarshal operation +func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error { + type Alias Template + alias := &Alias{} + err := unmarshal(alias) + if err != nil { + return err + } + *template = Template(*alias) + return validate.New().Struct(template) +} + +// MarshalJSON forces recursive struct validation during marshal operation +func (template *Template) MarshalJSON() ([]byte, error) { + out, marshalErr := json.Marshal(template) + errValidate := validate.New().Struct(template) + return out, multierr.Append(marshalErr, errValidate) +} + +// UnmarshalJSON forces recursive struct validation after unmarshal operation +func (template *Template) UnmarshalJSON(data []byte) error { + type Alias Template + alias := &Alias{} + err := json.Unmarshal(data, alias) + if err != nil { + return err + } + *template = Template(*alias) + return validate.New().Struct(template) +} From 07e7d0795b13697a20813bc0c0d0db0213a4cd05 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Dec 2021 11:51:06 +0100 Subject: [PATCH 20/41] Extending deny list to support filenames and folders (#1260) * Extending deny list to support filenames and folders * fixing field name * adding missing edge case with relative path + filename * handling root path + relative path * Improving matchers to handle all deny cases --- v2/pkg/protocols/file/file.go | 31 ++++----- v2/pkg/protocols/file/file_test.go | 12 ++-- v2/pkg/protocols/file/find.go | 86 +++++++++++++++++++++++-- v2/pkg/protocols/file/find_test.go | 10 +-- v2/pkg/protocols/file/operators_test.go | 40 ++++++------ v2/pkg/protocols/file/request_test.go | 10 +-- 6 files changed, 131 insertions(+), 58 deletions(-) diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index b0902e7a0..afb8b3da5 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -1,6 +1,7 @@ package file import ( + "path/filepath" "strings" "github.com/pkg/errors" @@ -19,13 +20,13 @@ type Request struct { // - value: '[]string{".txt", ".go", ".json"}' Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"` // description: | - // ExtensionDenylist is the list of file extensions to deny during matching. + // DenyList is the list of file, directories or extensions to deny during matching. // // By default, it contains some non-interesting extensions that are hardcoded // in nuclei. // examples: // - value: '[]string{".avi", ".mov", ".mp3"}' - ExtensionDenylist []string `yaml:"denylist,omitempty" jsonschema:"title=extensions to deny match,description=List of file extensions to deny during matching"` + DenyList []string `yaml:"denylist,omitempty" jsonschema:"title=denylist, directories and extentions to deny match,description=List of files, directories and extensions to deny during matching"` // ID is the optional id of the request ID string `yaml:"id,omitempty" jsonschema:"title=id of the request,description=ID is the optional ID for the request"` @@ -41,9 +42,9 @@ type Request struct { CompiledOperators *operators.Operators `yaml:"-"` // cache any variables that may be needed for operation. - options *protocols.ExecuterOptions - extensions map[string]struct{} - extensionDenylist map[string]struct{} + options *protocols.ExecuterOptions + extensions map[string]struct{} + denyList map[string]struct{} // description: | // NoRecursive specifies whether to not do recursive checks if folders are provided. @@ -89,7 +90,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.options = options request.extensions = make(map[string]struct{}) - request.extensionDenylist = make(map[string]struct{}) + request.denyList = make(map[string]struct{}) for _, extension := range request.Extensions { if extension == "all" { @@ -101,17 +102,17 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.extensions[extension] = struct{}{} } } - for _, extension := range defaultDenylist { - if !strings.HasPrefix(extension, ".") { - extension = "." + extension + // process default denylist (extensions) + for _, excludeItem := range defaultDenylist { + if !strings.HasPrefix(excludeItem, ".") { + excludeItem = "." + excludeItem } - request.extensionDenylist[extension] = struct{}{} + request.denyList[excludeItem] = struct{}{} } - for _, extension := range request.ExtensionDenylist { - if !strings.HasPrefix(extension, ".") { - extension = "." + extension - } - request.extensionDenylist[extension] = struct{}{} + for _, excludeItem := range request.DenyList { + request.denyList[excludeItem] = struct{}{} + // also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d) + request.denyList[filepath.Clean(excludeItem)] = struct{}{} } return nil } diff --git a/v2/pkg/protocols/file/file_test.go b/v2/pkg/protocols/file/file_test.go index d568f9810..c32b7a313 100644 --- a/v2/pkg/protocols/file/file_test.go +++ b/v2/pkg/protocols/file/file_test.go @@ -16,11 +16,11 @@ func TestFileCompile(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"all", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"all", ".lock"}, + DenyList: []string{".go"}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -29,7 +29,7 @@ func TestFileCompile(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") - require.Contains(t, request.extensionDenylist, ".go", "could not get .go in denylist") + require.Contains(t, request.denyList, ".go", "could not get .go in denylist") require.NotContains(t, request.extensions, ".go", "could get .go in allowlist") require.True(t, request.allExtensions, "could not get correct allExtensions") } diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index f4deaa100..63b1597c8 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -7,7 +7,7 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" - + "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/gologger" ) @@ -51,7 +51,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string return errors.Errorf("wildcard found, but unable to glob: %s\n", err) } for _, match := range matches { - if !request.validatePath(match) { + if !request.validatePath(absPath, match) { continue } if _, ok := processed[match]; !ok { @@ -73,7 +73,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str return false, nil } if _, ok := processed[absPath]; !ok { - if !request.validatePath(absPath) { + if !request.validatePath(absPath, absPath) { return false, nil } processed[absPath] = struct{}{} @@ -93,7 +93,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin if d.IsDir() { return nil } - if !request.validatePath(path) { + if !request.validatePath(absPath, path) { return nil } if _, ok := processed[path]; !ok { @@ -107,7 +107,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin } // validatePath validates a file path for blacklist and whitelist options -func (request *Request) validatePath(item string) bool { +func (request *Request) validatePath(absPath, item string) bool { extension := filepath.Ext(item) if len(request.extensions) > 0 { @@ -117,9 +117,81 @@ func (request *Request) validatePath(item string) bool { return false } } - if _, ok := request.extensionDenylist[extension]; ok { - gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension) + if matchingRule, ok := request.isInDenyList(absPath, item); ok { + gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule) return false } + return true } + +func (request *Request) isInDenyList(absPath, item string) (string, bool) { + extension := filepath.Ext(item) + // check for possible deny rules + // - extension is in deny list + if _, ok := request.denyList[extension]; ok { + return extension, true + } + + // - full path is in deny list + if _, ok := request.denyList[item]; ok { + return item, true + } + + // file is in a forbidden subdirectory + filename := filepath.Base(item) + fullPathWithoutFilename := strings.TrimSuffix(item, filename) + relativePathWithFilename := strings.TrimPrefix(item, absPath) + relativePath := strings.TrimSuffix(relativePathWithFilename, filename) + + // - filename is in deny list + if _, ok := request.denyList[filename]; ok { + return filename, true + } + + // - relative path is in deny list + if _, ok := request.denyList[relativePath]; ok { + return relativePath, true + } + + // relative path + filename are in the forbidden list + if _, ok := request.denyList[relativePathWithFilename]; ok { + return relativePathWithFilename, true + } + + // root path + relative path are in the forbidden list + if _, ok := request.denyList[fullPathWithoutFilename]; ok { + return fullPathWithoutFilename, true + } + + // check any progressive combined part of the relative and absolute path with filename for matches within rules prefixes + if pathTreeItem, ok := request.isAnyChunkInDenyList(relativePath, false); ok { + return pathTreeItem, true + } + if pathTreeItem, ok := request.isAnyChunkInDenyList(item, true); ok { + return pathTreeItem, true + } + + return "", false +} + +func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) { + var paths []string + + if splitWithUtils { + pathInfo, _ := folderutil.NewPathInfo(path) + paths, _ = pathInfo.Paths() + } else { + pathTree := strings.Split(path, string(os.PathSeparator)) + for i := range pathTree { + paths = append(paths, filepath.Join(pathTree[:i]...)) + } + } + for _, pathTreeItem := range paths { + if _, ok := request.denyList[pathTreeItem]; ok { + return pathTreeItem, true + } + } + + return "", false +} diff --git a/v2/pkg/protocols/file/find_test.go b/v2/pkg/protocols/file/find_test.go index 58eb128b6..9ca543adc 100644 --- a/v2/pkg/protocols/file/find_test.go +++ b/v2/pkg/protocols/file/find_test.go @@ -19,11 +19,11 @@ func TestFindInputPaths(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"all", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"all", ".lock"}, + DenyList: []string{".go"}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 37cc62d52..cfafe5b50 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -20,11 +20,11 @@ func TestResponseToDSLMap(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"*", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"*", ".lock"}, + DenyList: []string{".go"}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -45,11 +45,11 @@ func TestFileOperatorMatch(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"*", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"*", ".lock"}, + DenyList: []string{".go"}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -133,11 +133,11 @@ func TestFileOperatorExtract(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"*", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"*", ".lock"}, + DenyList: []string{".go"}, } executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ ID: templateID, @@ -240,11 +240,11 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"*", ".lock"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"*", ".lock"}, + DenyList: []string{".go"}, Operators: operators.Operators{ MatchersCondition: matcherCondition, Matchers: matchers, diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index 076c3b884..2f720e59a 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -23,11 +23,11 @@ func TestFileExecuteWithResults(t *testing.T) { testutils.Init(options) templateID := "testing-file" request := &Request{ - ID: templateID, - MaxSize: 1024, - NoRecursive: false, - Extensions: []string{"all"}, - ExtensionDenylist: []string{".go"}, + ID: templateID, + MaxSize: 1024, + NoRecursive: false, + Extensions: []string{"all"}, + DenyList: []string{".go"}, Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", From 8b20f384a779511d45ea20e935c4d565ad39240e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 16 Dec 2021 10:52:36 +0000 Subject: [PATCH 21/41] Auto Generate Syntax Docs + JSONSchema [Thu Dec 16 10:52:36 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 2 +- nuclei-jsonschema.json | 4 ++-- v2/pkg/templates/templates_doc.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index ab5ea9ae9..5ca82edaa 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2561,7 +2561,7 @@ extensions:
-ExtensionDenylist is the list of file extensions to deny during matching. +DenyList is the list of file, directories or extensions to deny during matching. By default, it contains some non-interesting extensions that are hardcoded in nuclei. diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 156090895..d7de55a24 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -472,8 +472,8 @@ "type": "string" }, "type": "array", - "title": "extensions to deny match", - "description": "List of file extensions to deny during matching" + "title": "denylist", + "description": "List of files" }, "id": { "type": "string", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index e1b739e9f..a3adad887 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -1139,8 +1139,8 @@ func init() { FILERequestDoc.Fields[4].Name = "denylist" FILERequestDoc.Fields[4].Type = "[]string" FILERequestDoc.Fields[4].Note = "" - FILERequestDoc.Fields[4].Description = "ExtensionDenylist is the list of file extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei." - FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "ExtensionDenylist is the list of file extensions to deny during matching." + FILERequestDoc.Fields[4].Description = "DenyList is the list of file, directories or extensions to deny during matching.\n\nBy default, it contains some non-interesting extensions that are hardcoded\nin nuclei." + FILERequestDoc.Fields[4].Comments[encoder.LineComment] = "DenyList is the list of file, directories or extensions to deny during matching." FILERequestDoc.Fields[4].AddExample("", []string{".avi", ".mov", ".mp3"}) FILERequestDoc.Fields[5].Name = "id" From d15298648527f4edd1015e077201551b3a6eab06 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 16 Dec 2021 05:06:20 -0600 Subject: [PATCH 22/41] Change filepath to path (#1382) * filepath to path * change to revert * reverted change * update: disabling cached nuclei temporarily Co-authored-by: sandeep --- .github/workflows/template-validate.yml | 2 +- v2/pkg/protocols/http/raw/raw.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml index 0f78d4de6..c4c677d0e 100644 --- a/.github/workflows/template-validate.yml +++ b/.github/workflows/template-validate.yml @@ -19,7 +19,7 @@ jobs: key: ${{ runner.os }}-go - name: Installing Nuclei - if: steps.cache-go.outputs.cache-hit != 'true' +# if: steps.cache-go.outputs.cache-hit != 'true' run: | go install github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 60db9b387..0dadc66f3 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -8,7 +8,7 @@ import ( "io" "io/ioutil" "net/url" - "path/filepath" + "path" "strings" "github.com/projectdiscovery/rawhttp/client" @@ -30,7 +30,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { rawRequest := &Request{ Headers: make(map[string]string), } - + parsedURL, err := url.Parse(baseURL) if err != nil { return nil, fmt.Errorf("could not parse request URL: %w", err) @@ -143,7 +143,7 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { } func fixUnsafeRequestPath(baseURL *url.URL, requestPath string, request []byte) []byte { - fixedPath := filepath.Join(baseURL.Path, requestPath) + fixedPath := path.Join(baseURL.Path, requestPath) fixed := bytes.Replace(request, []byte(requestPath), []byte(fixedPath), 1) return fixed } From 5200bcd94f7ca146ea5c1408965762b883cf847d Mon Sep 17 00:00:00 2001 From: Sajad Date: Thu, 16 Dec 2021 17:08:02 +0530 Subject: [PATCH 23/41] Whois Protocol Support (using rdap) (#1354) * init rdap * add an integration test, option to supply RDAP server to execute the request on * add rdap protocolMappings * add debug info, add IP, ASN query type support * rename rdap to whois, Host to Query in template * rename pending rdap to whois * remove port from whois varaiables * set Host variable even if input is not a parsable url --- integration_tests/whois/basic.yaml | 14 ++ v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/whois.go | 23 +++ v2/go.mod | 5 + v2/go.sum | 6 + v2/pkg/operators/matchers/match.go | 2 +- v2/pkg/protocols/whois/whois.go | 192 ++++++++++++++++++++ v2/pkg/templates/compile.go | 6 +- v2/pkg/templates/templates.go | 7 + v2/pkg/templates/types/types.go | 3 + 10 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 integration_tests/whois/basic.yaml create mode 100644 v2/cmd/integration-test/whois.go create mode 100644 v2/pkg/protocols/whois/whois.go diff --git a/integration_tests/whois/basic.yaml b/integration_tests/whois/basic.yaml new file mode 100644 index 000000000..742272a9a --- /dev/null +++ b/integration_tests/whois/basic.yaml @@ -0,0 +1,14 @@ +id: basic-whois-example + +info: + name: test template for WHOIS + author: pdteam + severity: info + +whois: + - query: "{{Host}}" + extractors: + - type: kval + kval: + - "expiration date" + - "registrar" \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index be9ab4c16..4bbed10a1 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -26,6 +26,7 @@ var ( "loader": loaderTestcases, "websocket": websocketTestCases, "headless": headlessTestcases, + "whois": whoisTestCases, } ) diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go new file mode 100644 index 000000000..3935f40e8 --- /dev/null +++ b/v2/cmd/integration-test/whois.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var whoisTestCases = map[string]testutils.TestCase{ + "whois/basic.yaml": &whoisBasic{}, +} + +type whoisBasic struct{} + +// Execute executes a test case and returns an error if occurred +func (h *whoisBasic) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://example.com", debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/v2/go.mod b/v2/go.mod index 41dbc4d87..2d0a8d9c6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -75,6 +75,8 @@ require ( github.com/PuerkitoBio/goquery v1.6.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.2.0 // indirect @@ -111,8 +113,10 @@ require ( github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/openrdap/rdap v0.9.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect @@ -133,6 +137,7 @@ require ( golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 3df87760e..f53abb112 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -71,8 +71,10 @@ github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= @@ -501,6 +503,7 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -551,6 +554,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7 github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/openrdap/rdap v0.9.0 h1:qaicAQD7XLPH8k4B8FQ6OnBL10joHt2xTREFA38T+wA= +github.com/openrdap/rdap v0.9.0/go.mod h1:Z9b01CBKIgQsX1JzmmhkhoQfdWYchgCP6B4QfKXSdD0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1209,6 +1214,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 898eaca0a..052e2c856 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -154,7 +154,7 @@ func (matcher *Matcher) MatchBinary(corpus string) (bool, []string) { // MatchDSL matches on a generic map result func (matcher *Matcher) MatchDSL(data map[string]interface{}) bool { - logExpressionEvaluationFailure := func (matcherName string, err error) { + logExpressionEvaluationFailure := func(matcherName string, err error) { gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", matcherName, err.Error()) } diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go new file mode 100644 index 000000000..6da0305f5 --- /dev/null +++ b/v2/pkg/protocols/whois/whois.go @@ -0,0 +1,192 @@ +package whois + +import ( + "net/url" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/openrdap/rdap" + "github.com/pkg/errors" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Request is a request for the WHOIS protocol +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // description: | + // Query contains query for the request + Query string `yaml:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` + + // description: | + // Optional WHOIS server URL. + // + // If present, specifies the WHOIS server to execute the Request on. + // Otherwise, nil enables bootstrapping + Server string `yaml:"server,omitempty" jsonschema:"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on"` + // cache any variables that may be needed for operation. + client *rdap.Client + options *protocols.ExecuterOptions + parsedServerURL *url.URL +} + +// Compile compiles the request generators preparing any requests possible. +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + var err error + if request.Server != "" { + request.parsedServerURL, err = url.Parse(request.Server) + if err != nil { + return errors.Wrap(err, "failed to parse server URL") + } + } + + request.options = options + request.client = &rdap.Client{} + + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + request.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the rule will perform +func (request *Request) Requests() int { + return 1 +} + +// GetID returns the ID for the request if any. +func (request *Request) GetID() string { + return "" +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + // generate variables + variables := generateVariables(input) + // and replace placeholders + query := replacer.Replace(request.Query, variables) + // build an rdap request + rdapReq := rdap.NewAutoRequest(query) + res, err := request.client.Do(rdapReq) + if err != nil { + return errors.Wrap(err, "could not make whois request") + } + gologger.Verbose().Msgf("Sent WHOIS request to %s", query) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Dumped WHOIS request for %s", request.options.TemplateID, query) + } + + data := make(map[string]interface{}) + var response interface{} + switch rdapReq.Type { + case rdap.DomainRequest: + // convert the rdap response to a whois style response (for domain request type only) + whoisResp := res.ToWhoisStyleResponse() + for k, v := range whoisResp.Data { + data[strings.ToLower(k)] = strings.Join(v, ",") + } + response = whoisResp + default: + response = res.Object + } + jsonData, _ := jsoniter.Marshal(response) + jsonDataString := string(jsonData) + + data["type"] = request.Type().String() + data["host"] = query + data["response"] = jsonDataString + + event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false)) + } + + callback(event) + return nil +} + +// Match performs matching operation for a matcher on model and returns: +// true and a list of matched snippets if the matcher type is supports it +// otherwise false and an empty string slice +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { + return protocols.MakeDefaultMatchFunc(data, matcher) +} + +// Extract performs extracting operation for an extractor on model and returns true or false. +func (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { + return protocols.MakeDefaultExtractFunc(data, matcher) +} + +// MakeResultEvent creates a result event from internal wrapped event +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) +} + +// GetCompiledOperators returns a list of the compiled operators +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(request.options.TemplateID), + TemplatePath: types.ToString(request.options.TemplatePath), + Info: request.options.TemplateInfo, + Type: types.ToString(wrapped.InternalEvent["type"]), + Host: types.ToString(wrapped.InternalEvent["host"]), + Metadata: wrapped.OperatorsResult.PayloadValues, + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + MatcherStatus: true, + Request: types.ToString(wrapped.InternalEvent["request"]), + Response: types.ToString(wrapped.InternalEvent["response"]), + } + return data +} + +// Type returns the type of the protocol request +func (request *Request) Type() templateTypes.ProtocolType { + return templateTypes.WHOISProtocol +} + +// generateVariables will create default variables after parsing a url +func generateVariables(input string) map[string]interface{} { + var domain string + + parsed, err := url.Parse(input) + if err != nil { + return map[string]interface{}{"Input": input} + } + domain = parsed.Host + if domain == "" { + domain = input + } + if strings.Contains(domain, ":") { + domain = strings.Split(domain, ":")[0] + } + + return map[string]interface{}{ + "Input": input, + "Hostname": parsed.Host, + "Host": domain, + } +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 079c96f4a..84c56acd8 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -128,7 +128,8 @@ func (template *Template) Requests() int { len(template.RequestsHeadless) + len(template.Workflows) + len(template.RequestsSSL) + - len(template.RequestsWebsocket) + len(template.RequestsWebsocket) + + len(template.RequestsWHOIS) } // compileProtocolRequests compiles all the protocol requests for the template @@ -166,6 +167,9 @@ func (template *Template) compileProtocolRequests(options protocols.ExecuterOpti case len(template.RequestsWebsocket) > 0: requests = template.convertRequestToProtocolsRequest(template.RequestsWebsocket) + + case len(template.RequestsWHOIS) > 0: + requests = template.convertRequestToProtocolsRequest(template.RequestsWHOIS) } template.Executer = executer.NewExecuter(requests, &options) return nil diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index bdf595cf0..d8bacffd8 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/whois" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "go.uber.org/multierr" @@ -71,6 +72,9 @@ type Template struct { // Websocket contains the Websocket request to make in the template. RequestsWebsocket []*websocket.Request `yaml:"websocket,omitempty" json:"websocket,omitempty" jsonschema:"title=websocket requests to make,description=Websocket requests to make for the template"` + // description: | + // WHOIS contains the WHOIS request to make in the template. + RequestsWHOIS []*whois.Request `yaml:"whois,omitempty" json:"whois,omitempty" jsonschema:"title=whois requests to make,description=WHOIS requests to make for the template"` // description: | // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"` @@ -101,6 +105,7 @@ var TemplateProtocols = []string{ "workflow", "ssl", "websocket", + "whois", } // Type returns the type of the template @@ -122,6 +127,8 @@ func (template *Template) Type() types.ProtocolType { return types.SSLProtocol case len(template.RequestsWebsocket) > 0: return types.WebsocketProtocol + case len(template.RequestsWHOIS) > 0: + return types.WHOISProtocol default: return types.InvalidProtocol } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index be6b72453..cc0c8b93d 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -34,6 +34,8 @@ const ( SSLProtocol // name:websocket WebsocketProtocol + // name:whois + WHOISProtocol limit InvalidProtocol ) @@ -49,6 +51,7 @@ var protocolMappings = map[ProtocolType]string{ WorkflowProtocol: "workflow", SSLProtocol: "ssl", WebsocketProtocol: "websocket", + WHOISProtocol: "whois", } func GetSupportedProtocolTypes() ProtocolTypes { From 4c70e4976bb1d5b2d5de4e8a8c1184c98e24a407 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 16 Dec 2021 11:39:23 +0000 Subject: [PATCH 24/41] Auto Generate Syntax Docs + JSONSchema [Thu Dec 16 11:39:23 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 117 ++++++++++++++++++++++++++++++ nuclei-jsonschema.json | 50 +++++++++++++ v2/pkg/templates/templates_doc.go | 79 +++++++++++++++++--- 3 files changed, 234 insertions(+), 12 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 5ca82edaa..a1ebbc87e 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -247,6 +247,19 @@ Websocket contains the Websocket request to make in the template.
+whois []whois.Request + +
+
+ +WHOIS contains the WHOIS request to make in the template. + +
+ +
+ +
+ workflows []workflows.WorkflowTemplate
@@ -1320,6 +1333,8 @@ Appears in: - websocket.Request.matchers +- whois.Request.matchers + @@ -1713,6 +1728,8 @@ Appears in: - websocket.Request.extractors +- whois.Request.extractors + @@ -3599,6 +3616,106 @@ name: prefix +## whois.Request +Request is a request for the WHOIS protocol + +Appears in: + + +- Template.whois + + + + + +
+ +
+ +matchers []matchers.Matcher + +
+
+ +Matchers contains the detection mechanism for the request to identify +whether the request was successful by doing pattern matching +on request/responses. + +Multiple matchers can be combined with `matcher-condition` flag +which accepts either `and` or `or` as argument. + +
+ +
+ +
+ +extractors []extractors.Extractor + +
+
+ +Extractors contains the extraction mechanism for the request to identify +and extract parts of the response. + +
+ +
+ +
+ +matchers-condition string + +
+
+ +MatchersCondition is the condition between the matchers. Default is OR. + + +Valid values: + + + - and + + - or +
+ +
+ +
+ +query string + +
+
+ +Query contains query for the request + +
+ +
+ +
+ +server string + +
+
+ +description: | + Optional WHOIS server URL. + + If present, specifies the WHOIS server to execute the Request on. + Otherwise, nil enables bootstrapping + +
+ +
+ + + + + ## workflows.WorkflowTemplate Appears in: diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index d7de55a24..c62995d4f 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -1026,6 +1026,47 @@ "additionalProperties": false, "type": "object" }, + "whois.Request": { + "properties": { + "matchers": { + "items": { + "$ref": "#/definitions/matchers.Matcher" + }, + "type": "array", + "title": "matchers to run on response", + "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + }, + "extractors": { + "items": { + "$ref": "#/definitions/extractors.Extractor" + }, + "type": "array", + "title": "extractors to run on response", + "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + }, + "matchers-condition": { + "enum": [ + "and", + "or" + ], + "type": "string", + "title": "condition between the matchers", + "description": "Conditions between the matchers" + }, + "query": { + "type": "string", + "title": "query for the WHOIS request", + "description": "Query contains query for the request" + }, + "server": { + "type": "string", + "title": "server url to execute the WHOIS request on", + "description": "Server contains the server url to execute the WHOIS request on" + } + }, + "additionalProperties": false, + "type": "object" + }, "templates.Template": { "required": [ "id", @@ -1110,6 +1151,15 @@ "title": "websocket requests to make", "description": "Websocket requests to make for the template" }, + "whois": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/whois.Request" + }, + "type": "array", + "title": "whois requests to make", + "description": "WHOIS requests to make for the template" + }, "workflows": { "items": { "$schema": "http://json-schema.org/draft-04/schema#", diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index a3adad887..962eba32c 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -33,6 +33,7 @@ var ( SSLRequestDoc encoder.Doc WEBSOCKETRequestDoc encoder.Doc WEBSOCKETInputDoc encoder.Doc + WHOISRequestDoc encoder.Doc WORKFLOWSWorkflowTemplateDoc encoder.Doc WORKFLOWSMatcherDoc encoder.Doc ) @@ -41,7 +42,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 12) + TemplateDoc.Fields = make([]encoder.Doc, 13) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -99,21 +100,26 @@ func init() { TemplateDoc.Fields[8].Note = "" TemplateDoc.Fields[8].Description = "Websocket contains the Websocket request to make in the template." TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Websocket contains the Websocket request to make in the template." - TemplateDoc.Fields[9].Name = "workflows" - TemplateDoc.Fields[9].Type = "[]workflows.WorkflowTemplate" + TemplateDoc.Fields[9].Name = "whois" + TemplateDoc.Fields[9].Type = "[]whois.Request" TemplateDoc.Fields[9].Note = "" - TemplateDoc.Fields[9].Description = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[9].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." - TemplateDoc.Fields[10].Name = "self-contained" - TemplateDoc.Fields[10].Type = "bool" + TemplateDoc.Fields[9].Description = "WHOIS contains the WHOIS request to make in the template." + TemplateDoc.Fields[9].Comments[encoder.LineComment] = "WHOIS contains the WHOIS request to make in the template." + TemplateDoc.Fields[10].Name = "workflows" + TemplateDoc.Fields[10].Type = "[]workflows.WorkflowTemplate" TemplateDoc.Fields[10].Note = "" - TemplateDoc.Fields[10].Description = "Self Contained marks Requests for the template as self-contained" - TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" - TemplateDoc.Fields[11].Name = "stop-at-first-match" + TemplateDoc.Fields[10].Description = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[10].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[11].Name = "self-contained" TemplateDoc.Fields[11].Type = "bool" TemplateDoc.Fields[11].Note = "" - TemplateDoc.Fields[11].Description = "Stop execution once first match is found" - TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Stop execution once first match is found" + TemplateDoc.Fields[11].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[11].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[12].Name = "stop-at-first-match" + TemplateDoc.Fields[12].Type = "bool" + TemplateDoc.Fields[12].Note = "" + TemplateDoc.Fields[12].Description = "Stop execution once first match is found" + TemplateDoc.Fields[12].Comments[encoder.LineComment] = "Stop execution once first match is found" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -570,6 +576,10 @@ func init() { TypeName: "websocket.Request", FieldName: "matchers", }, + { + TypeName: "whois.Request", + FieldName: "matchers", + }, } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 13) MATCHERSMatcherDoc.Fields[0].Name = "type" @@ -731,6 +741,10 @@ func init() { TypeName: "websocket.Request", FieldName: "extractors", }, + { + TypeName: "whois.Request", + FieldName: "extractors", + }, } EXTRACTORSExtractorDoc.Fields = make([]encoder.Doc, 11) EXTRACTORSExtractorDoc.Fields[0].Name = "name" @@ -1645,6 +1659,46 @@ func init() { WEBSOCKETInputDoc.Fields[1].AddExample("", "prefix") + WHOISRequestDoc.Type = "whois.Request" + WHOISRequestDoc.Comments[encoder.LineComment] = " Request is a request for the WHOIS protocol" + WHOISRequestDoc.Description = "Request is a request for the WHOIS protocol" + WHOISRequestDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "whois", + }, + } + WHOISRequestDoc.Fields = make([]encoder.Doc, 5) + WHOISRequestDoc.Fields[0].Name = "matchers" + WHOISRequestDoc.Fields[0].Type = "[]matchers.Matcher" + WHOISRequestDoc.Fields[0].Note = "" + WHOISRequestDoc.Fields[0].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + WHOISRequestDoc.Fields[0].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + WHOISRequestDoc.Fields[1].Name = "extractors" + WHOISRequestDoc.Fields[1].Type = "[]extractors.Extractor" + WHOISRequestDoc.Fields[1].Note = "" + WHOISRequestDoc.Fields[1].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + WHOISRequestDoc.Fields[1].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + WHOISRequestDoc.Fields[2].Name = "matchers-condition" + WHOISRequestDoc.Fields[2].Type = "string" + WHOISRequestDoc.Fields[2].Note = "" + WHOISRequestDoc.Fields[2].Description = "MatchersCondition is the condition between the matchers. Default is OR." + WHOISRequestDoc.Fields[2].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + WHOISRequestDoc.Fields[2].Values = []string{ + "and", + "or", + } + WHOISRequestDoc.Fields[3].Name = "query" + WHOISRequestDoc.Fields[3].Type = "string" + WHOISRequestDoc.Fields[3].Note = "" + WHOISRequestDoc.Fields[3].Description = "Query contains query for the request" + WHOISRequestDoc.Fields[3].Comments[encoder.LineComment] = "Query contains query for the request" + WHOISRequestDoc.Fields[4].Name = "server" + WHOISRequestDoc.Fields[4].Type = "string" + WHOISRequestDoc.Fields[4].Note = "" + WHOISRequestDoc.Fields[4].Description = "description: |\n Optional WHOIS server URL.\n\n If present, specifies the WHOIS server to execute the Request on.\n Otherwise, nil enables bootstrapping" + WHOISRequestDoc.Fields[4].Comments[encoder.LineComment] = " description: |" + WORKFLOWSWorkflowTemplateDoc.Type = "workflows.WorkflowTemplate" WORKFLOWSWorkflowTemplateDoc.Comments[encoder.LineComment] = "" WORKFLOWSWorkflowTemplateDoc.Description = "" @@ -1740,6 +1794,7 @@ func GetTemplateDoc() *encoder.FileDoc { &SSLRequestDoc, &WEBSOCKETRequestDoc, &WEBSOCKETInputDoc, + &WHOISRequestDoc, &WORKFLOWSWorkflowTemplateDoc, &WORKFLOWSMatcherDoc, }, From 714f0c82a9edbe22a77b1a0a6ccd52f79cdc4404 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 16 Dec 2021 23:41:18 +0100 Subject: [PATCH 25/41] adding missing return error --- v2/pkg/protocols/http/build_request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index b8d60babb..0e474ba80 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -125,7 +125,7 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam // the url might contain placeholders parts[1] = replacer.Replace(parts[1], generators.BuildPayloadFromOptions(r.request.options.Options)) - if expressions.ContainsUnresolvedVariables(parts[1]) != nil { + if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { return nil, err } From e9034a459dee8ac26f90d7d62ffbd2a377f5e4e4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 17 Dec 2021 07:25:46 +0100 Subject: [PATCH 26/41] Fixing wrong function/variable name in whois integration test (#1389) * Fixing wrong function/variable name * remove explicit result length check for whois integration test Co-authored-by: Sajad Parra --- v2/cmd/integration-test/whois.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/v2/cmd/integration-test/whois.go b/v2/cmd/integration-test/whois.go index 3935f40e8..edb534e43 100644 --- a/v2/cmd/integration-test/whois.go +++ b/v2/cmd/integration-test/whois.go @@ -16,8 +16,5 @@ func (h *whoisBasic) Execute(filePath string) error { if err != nil { return err } - if len(results) != 1 { - return errIncorrectResultsCount(results) - } - return nil + return expectResultsCount(results, 1) } From 860a40717cafb0fc34a25bd066ff690177ba3654 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 18 Dec 2021 13:59:13 +0100 Subject: [PATCH 27/41] Removing outdated macos-11.0 from gh action (#1390) --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2de3eec15..051a52c4b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest, macos-11.0] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go uses: actions/setup-go@v2 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 9ea360751..ad2b95610 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest, macos-11.0] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go uses: actions/setup-go@v2 From 6a408fd6a8862ca7f8ebebc54534b9e177244681 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sat, 18 Dec 2021 19:32:11 +0530 Subject: [PATCH 28/41] feature: exposed interaction ip information to matchers/extractors (#1395) --- v2/pkg/protocols/common/interactsh/interactsh.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index a821f46ad..2781d8f52 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -165,6 +165,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse + data.Event.InternalEvent["interactsh_ip"] = interaction.RemoteAddress result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false) if !matched || result == nil { return false // if we don't match, return From 1297c883a604ec7b2491416584388526302b9b8c Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 18 Dec 2021 19:43:52 +0530 Subject: [PATCH 29/41] version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 0ac5a5ffd..e1eaab253 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.6-dev` +const Version = `2.5.6` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 2f00e27bd10a116ad70818688afc89e231ad5545 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 18 Dec 2021 19:58:59 +0530 Subject: [PATCH 30/41] dev version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index e1eaab253..ff880403d 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.6` +const Version = `2.5.7-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From e59da293710ec5afe3a954def7d7afa822da9eb4 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 18 Dec 2021 20:06:51 +0100 Subject: [PATCH 31/41] improving error/args handling --- .../protocols/common/expressions/variables.go | 57 ++++++++++++------- v2/pkg/protocols/http/build_request.go | 22 +++++-- v2/pkg/protocols/http/request.go | 18 ++++-- v2/pkg/protocols/http/signature.go | 23 ++++++++ v2/pkg/protocols/http/signer/aws.go | 17 ++++-- v2/pkg/protocols/http/signer/signer.go | 8 +-- 6 files changed, 105 insertions(+), 40 deletions(-) diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go index 2aba5c324..e0a0879a8 100644 --- a/v2/pkg/protocols/common/expressions/variables.go +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -16,49 +16,66 @@ func ContainsUnresolvedVariables(items ...string) error { if len(matches) == 0 { return nil } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables found: ") - - for i, match := range matches { + var unresolvedVariables []string + for _, match := range matches { if len(match) < 2 { continue } - errorString.WriteString(match[1]) - if i != len(matches)-1 { - errorString.WriteString(",") - } + unresolvedVariables = append(unresolvedVariables, match[1]) } - errorMessage := errorString.String() - return errors.New(errorMessage) + return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ",")) } return nil } +// ContainsVariablesWithNames returns an error with variable names if the passed +// input contains unresolved {{}} variables within the provided list func ContainsVariablesWithNames(names map[string]interface{}, items ...string) error { for _, data := range items { matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) if len(matches) == 0 { return nil } - errorString := &strings.Builder{} - errorString.WriteString("unresolved variables with values found: ") - - for i, match := range matches { + var unresolvedVariables []string + for _, match := range matches { if len(match) < 2 { continue } matchName := match[1] if _, ok := names[matchName]; !ok { - errorString.WriteString(matchName) - if i != len(matches)-1 { - errorString.WriteString(",") - } + unresolvedVariables = append(unresolvedVariables, matchName) } } - errorMessage := errorString.String() - return errors.New(errorMessage) + return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ",")) } return nil } + +// ContainsVariablesWithIgnoreList returns an error with variable names if the passed +// input contains unresolved {{}} other than the ones listed in the ignore list +func ContainsVariablesWithIgnoreList(skipNames map[string]interface{}, items ...string) error { + var unresolvedVariables []string + for _, data := range items { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + for _, match := range matches { + if len(match) < 2 { + continue + } + matchName := match[1] + if _, ok := skipNames[matchName]; ok { + continue + } + unresolvedVariables = append(unresolvedVariables, matchName) + } + } + + if len(unresolvedVariables) > 0 { + return errors.New("unresolved variables with values found: " + strings.Join(unresolvedVariables, ",")) + } + return nil +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 0e474ba80..70e212c70 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -123,10 +123,22 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam return nil, fmt.Errorf("malformed request supplied") } - // the url might contain placeholders - parts[1] = replacer.Replace(parts[1], generators.BuildPayloadFromOptions(r.request.options.Options)) - if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { - return nil, err + payloads := generators.BuildPayloadFromOptions(r.request.options.Options) + // in case cases (eg requests signing, some variables uses default values if missing) + if defaultList := GetVariablesDefault(r.request.Signature.Value); defaultList != nil { + payloads = generators.MergeMaps(defaultList, payloads) + } + parts[1] = replacer.Replace(parts[1], payloads) + + // the url might contain placeholders with ignore list + if ignoreList := GetVariablesNamesSkipList(r.request.Signature.Value); ignoreList != nil { + if err := expressions.ContainsVariablesWithIgnoreList(ignoreList, parts[1]); err != nil { + return nil, err + } + } else { // the url might contain placeholders + if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil { + return nil, err + } } parsed, err := url.Parse(parts[1]) @@ -135,7 +147,7 @@ func (r *requestGenerator) makeSelfContainedRequest(data string, payloads, dynam } values := generators.MergeMaps( generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), - generators.BuildPayloadFromOptions(r.request.options.Options), + payloads, ) return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 11902d67d..37987adf5 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -350,10 +350,16 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } dumpedRequestString := string(dumpedRequest) - // Check if are there any unresolved variables. If yes, skip unless overridden by user. - if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) - return errStopExecution + if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { + if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) + return errStopExecution + } + } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. + if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) + return errStopExecution + } } } var formedURL string @@ -562,9 +568,9 @@ func (request *Request) handleSignature(generatedRequest *generatedRequest) erro awsSignerArgs := signer.AwsSignerArgs{AwsId: awsAccessKeyId, AwsSecretToken: awsSecretAccessKey} service := types.ToString(payloads["service"]) region := types.ToString(payloads["region"]) - // if region is empty default to "us-east-2" + // if region is empty use default value if region == "" { - region = "us-east-2" + region = types.ToString(signer.AwsDefaultVars["region"]) } awsSignatureArguments := signer.AwsSignatureArguments{ Service: types.ToString(service), diff --git a/v2/pkg/protocols/http/signature.go b/v2/pkg/protocols/http/signature.go index 53f31164f..290c628e2 100644 --- a/v2/pkg/protocols/http/signature.go +++ b/v2/pkg/protocols/http/signature.go @@ -5,6 +5,7 @@ import ( "github.com/alecthomas/jsonschema" "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer" ) // SignatureType is the type of signature @@ -82,3 +83,25 @@ func (holder *SignatureTypeHolder) MarshalJSON() ([]byte, error) { func (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) { return holder.Value.String(), nil } + +var ErrNoIgnoreList = errors.New("uknown signature types") + +// GetVariablesNamesSkipList depending on the signature type +func GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} { + switch signature { + case AWSSignature: + return signer.AwsSkipList + default: + return nil + } +} + +// GetVariablesNamesSkipList depending on the signature type +func GetVariablesDefault(signature SignatureType) map[string]interface{} { + switch signature { + case AWSSignature: + return signer.AwsDefaultVars + default: + return nil + } +} diff --git a/v2/pkg/protocols/http/signer/aws.go b/v2/pkg/protocols/http/signer/aws.go index ccd852c8d..3b6488712 100644 --- a/v2/pkg/protocols/http/signer/aws.go +++ b/v2/pkg/protocols/http/signer/aws.go @@ -58,9 +58,7 @@ func NewAwsSigner(args AwsSignerArgs) (*AwsSigner, error) { if creds == nil { return nil, errors.New("couldn't create the credentials structure") } - signer := v4.NewSigner(creds) - return &AwsSigner{creds: creds, signer: signer}, nil } @@ -69,7 +67,8 @@ func NewAwsSignerFromEnv() (*AwsSigner, error) { if creds == nil { return nil, errors.New("couldn't create the credentials structure") } - return &AwsSigner{creds: creds}, nil + signer := v4.NewSigner(creds) + return &AwsSigner{creds: creds, signer: signer}, nil } func NewAwsSignerFromFile() (*AwsSigner, error) { @@ -77,7 +76,8 @@ func NewAwsSignerFromFile() (*AwsSigner, error) { if creds == nil { return nil, errors.New("couldn't create the credentials structure") } - return &AwsSigner{creds: creds}, nil + signer := v4.NewSigner(creds) + return &AwsSigner{creds: creds, signer: signer}, nil } func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) error { @@ -96,7 +96,6 @@ func (awsSigner *AwsSigner) SignHTTP(request *http.Request, args interface{}) er request.Body.Close() body = bytes.NewReader(bodyBytes) } - if _, err := awsSigner.signer.Sign(request, body, signatureArgs.Service, signatureArgs.Region, signatureArgs.Time); err != nil { return err } @@ -131,3 +130,11 @@ func (awsSigner *AwsSigner) checkSignatureArgs(args interface{}) (AwsSignatureAr func (awsSigner *AwsSigner) prepareRequest(request *http.Request) { request.Header.Del("Host") } + +var AwsSkipList = map[string]interface{}{ + "region": struct{}{}, +} + +var AwsDefaultVars = map[string]interface{}{ + "region": "us-east-2", +} diff --git a/v2/pkg/protocols/http/signer/signer.go b/v2/pkg/protocols/http/signer/signer.go index 262d17bb6..132eddd11 100644 --- a/v2/pkg/protocols/http/signer/signer.go +++ b/v2/pkg/protocols/http/signer/signer.go @@ -23,11 +23,11 @@ func NewSigner(args SignerArgs) (signer Signer, err error) { case AwsSignerArgs: awsSigner, err := NewAwsSigner(signerArgs) if err != nil { - // env variables - awsSigner, err = NewAwsSignerFromEnv() + // $HOME/.aws/credentials + awsSigner, err = NewAwsSignerFromFile() if err != nil { - // $HOME/.aws/credentials - awsSigner, err = NewAwsSignerFromFile() + // env variables + awsSigner, err = NewAwsSignerFromEnv() if err != nil { return nil, err } From 0a82669163f0b421f6a419dde9245858b6c05ccd Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Sun, 19 Dec 2021 15:44:17 +0530 Subject: [PATCH 32/41] update rdap to fix go mod tidy error --- v2/go.mod | 8 ++++---- v2/go.sum | 16 ++++++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 2d0a8d9c6..e128e0533 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -56,7 +56,7 @@ require ( go.uber.org/atomic v1.9.0 go.uber.org/multierr v1.7.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20211020060615-d418f374d309 + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/text v0.3.7 gopkg.in/yaml.v2 v2.4.0 @@ -67,6 +67,7 @@ require github.com/projectdiscovery/folderutil v0.0.0-20211206150108-b4e7ea80f36 require ( github.com/Ice3man543/nvd v1.0.8 + github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/stretchr/testify v1.7.0 ) @@ -76,7 +77,7 @@ require ( github.com/StackExchange/wmi v1.2.1 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.2.0 // indirect @@ -116,7 +117,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/openrdap/rdap v0.9.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect @@ -132,7 +132,7 @@ require ( github.com/ysmood/goob v0.3.0 // indirect github.com/zclconf/go-cty v1.8.4 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index f53abb112..49594ca47 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -74,8 +74,9 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= @@ -404,6 +405,8 @@ github.com/itchyny/gojq v0.12.5 h1:6SJ1BQ1VAwJAlIvLSIZmqHP/RUEq3qfVWvsRxrqhsD0= github.com/itchyny/gojq v0.12.5/go.mod h1:3e1hZXv+Kwvdp6V9HXpVrvddiHVApi5EDZwS+zLFeiE= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= +github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -554,8 +557,8 @@ github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7 github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/openrdap/rdap v0.9.0 h1:qaicAQD7XLPH8k4B8FQ6OnBL10joHt2xTREFA38T+wA= -github.com/openrdap/rdap v0.9.0/go.mod h1:Z9b01CBKIgQsX1JzmmhkhoQfdWYchgCP6B4QfKXSdD0= +github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 h1:3Xn/CN6GVY+7mVuGgt5bfp0F9JwcWqnvwfb23Jf8Vxg= +github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7/go.mod h1:inRbqVxN7ri77yTJY3ZtGtKegIFa3Qnarh7Xp9P7LgY= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -859,8 +862,9 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -950,8 +954,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI= -golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 6cde5843b53104fbd6debd5f8ee5339708648743 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 19 Dec 2021 18:05:09 +0530 Subject: [PATCH 33/41] go mod updates --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- v2/pkg/catalog/config/config.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e128e0533..1cd048cce 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -33,7 +33,7 @@ require ( github.com/projectdiscovery/goflags v0.0.8-0.20211028121123-edf02bc05b1a github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.6 + github.com/projectdiscovery/interactsh v0.0.7 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20211006155443-c0a8d610a4df github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20211109182249-43d38df59660 diff --git a/v2/go.sum b/v2/go.sum index 49594ca47..a1fee4c4f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -618,8 +618,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1: github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= -github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c= -github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= +github.com/projectdiscovery/interactsh v0.0.7 h1:X/qvJZR+qz/G7M8oNkbDVNeTxAY4T7aRqFIsSoa5mfA= +github.com/projectdiscovery/interactsh v0.0.7/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index ff880403d..d4864d6b6 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.7-dev` +const Version = `2.5.7` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From 33e98386c931ca7df402a678af0c2ac269d66b10 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 19 Dec 2021 18:16:06 +0530 Subject: [PATCH 34/41] dev version update --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index d4864d6b6..7b6e6ab39 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.7` +const Version = `2.5.8-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() From fdc29180a0dbedd65e67c087139e4c0393891ebb Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 19 Dec 2021 19:37:37 +0000 Subject: [PATCH 35/41] Auto Generate Syntax Docs + JSONSchema [Sun Dec 19 19:37:37 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 64 +++++++++++++++++++ nuclei-jsonschema.json | 19 ++++++ v2/pkg/templates/templates_doc.go | 102 +++++++++++++++++++++--------- 3 files changed, 155 insertions(+), 30 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index a1ebbc87e..c0e8941bc 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -297,6 +297,24 @@ Stop execution once first match is found
+
+ +signature http.SignatureTypeHolder + +
+
+ +Signature is the request signature method + + +Valid values: + + + - AWS +
+ +
+ @@ -1182,6 +1200,24 @@ max-size: 2048
+signature SignatureTypeHolder + +
+
+ +Signature is the request signature method + + +Valid values: + + + - AWS +
+ +
+ +
+ cookie-reuse bool
@@ -2144,6 +2180,20 @@ Enum Values: +## SignatureTypeHolder +SignatureTypeHolder is used to hold internal type of the signature + +Appears in: + + +- http.Request.signature + + + + + + + ## dns.Request Request contains a DNS protocol request to be made from a template @@ -3847,3 +3897,17 @@ Subtemplates are run if the name of matcher matches. + +## http.SignatureTypeHolder +SignatureTypeHolder is used to hold internal type of the signature + +Appears in: + + +- Template.signature + + + + + + diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index c62995d4f..f600fcc1a 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -741,6 +741,12 @@ "title": "maximum http response body size", "description": "Maximum size of http response body to read in bytes" }, + "signature": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/http.SignatureTypeHolder", + "title": "signature is the http request signature method", + "description": "Signature is the HTTP Request signature Method" + }, "cookie-reuse": { "type": "boolean", "title": "optional cookie reuse enable", @@ -790,6 +796,14 @@ "additionalProperties": false, "type": "object" }, + "http.SignatureTypeHolder": { + "enum": [ + "AWS" + ], + "type": "string", + "title": "type of the signature", + "description": "Type of the signature" + }, "network.Input": { "properties": { "data": { @@ -1178,6 +1192,11 @@ "type": "boolean", "title": "stop at first match", "description": "Stop at first match for the template" + }, + "signature": { + "$ref": "#/definitions/http.SignatureTypeHolder", + "title": "signature is the http request signature method", + "description": "Signature is the HTTP Request signature Method" } }, "additionalProperties": false, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 962eba32c..4abe1612d 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -21,6 +21,7 @@ var ( ExtractorTypeHolderDoc encoder.Doc GENERATORSAttackTypeHolderDoc encoder.Doc HTTPMethodTypeHolderDoc encoder.Doc + SignatureTypeHolderDoc encoder.Doc DNSRequestDoc encoder.Doc DNSRequestTypeHolderDoc encoder.Doc FILERequestDoc encoder.Doc @@ -36,13 +37,14 @@ var ( WHOISRequestDoc encoder.Doc WORKFLOWSWorkflowTemplateDoc encoder.Doc WORKFLOWSMatcherDoc encoder.Doc + HTTPSignatureTypeHolderDoc encoder.Doc ) func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 13) + TemplateDoc.Fields = make([]encoder.Doc, 14) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -120,6 +122,14 @@ func init() { TemplateDoc.Fields[12].Note = "" TemplateDoc.Fields[12].Description = "Stop execution once first match is found" TemplateDoc.Fields[12].Comments[encoder.LineComment] = "Stop execution once first match is found" + TemplateDoc.Fields[13].Name = "signature" + TemplateDoc.Fields[13].Type = "http.SignatureTypeHolder" + TemplateDoc.Fields[13].Note = "" + TemplateDoc.Fields[13].Description = "Signature is the request signature method" + TemplateDoc.Fields[13].Comments[encoder.LineComment] = "Signature is the request signature method" + TemplateDoc.Fields[13].Values = []string{ + "AWS", + } MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -378,7 +388,7 @@ func init() { Value: "HTTP response headers in name:value format", }, } - HTTPRequestDoc.Fields = make([]encoder.Doc, 27) + HTTPRequestDoc.Fields = make([]encoder.Doc, 28) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[0].Note = "" @@ -498,51 +508,59 @@ func init() { HTTPRequestDoc.Fields[17].Comments[encoder.LineComment] = "MaxSize is the maximum size of http response body to read in bytes." HTTPRequestDoc.Fields[17].AddExample("Read max 2048 bytes of the response", 2048) - HTTPRequestDoc.Fields[18].Name = "cookie-reuse" - HTTPRequestDoc.Fields[18].Type = "bool" + HTTPRequestDoc.Fields[18].Name = "signature" + HTTPRequestDoc.Fields[18].Type = "SignatureTypeHolder" HTTPRequestDoc.Fields[18].Note = "" - HTTPRequestDoc.Fields[18].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section." - HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for" - HTTPRequestDoc.Fields[19].Name = "redirects" + HTTPRequestDoc.Fields[18].Description = "Signature is the request signature method" + HTTPRequestDoc.Fields[18].Comments[encoder.LineComment] = "Signature is the request signature method" + HTTPRequestDoc.Fields[18].Values = []string{ + "AWS", + } + HTTPRequestDoc.Fields[19].Name = "cookie-reuse" HTTPRequestDoc.Fields[19].Type = "bool" HTTPRequestDoc.Fields[19].Note = "" - HTTPRequestDoc.Fields[19].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects." - HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client." - HTTPRequestDoc.Fields[20].Name = "pipeline" + HTTPRequestDoc.Fields[19].Description = "CookieReuse is an optional setting that enables cookie reuse for\nall requests defined in raw section." + HTTPRequestDoc.Fields[19].Comments[encoder.LineComment] = "CookieReuse is an optional setting that enables cookie reuse for" + HTTPRequestDoc.Fields[20].Name = "redirects" HTTPRequestDoc.Fields[20].Type = "bool" HTTPRequestDoc.Fields[20].Note = "" - HTTPRequestDoc.Fields[20].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests." - HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining" - HTTPRequestDoc.Fields[21].Name = "unsafe" + HTTPRequestDoc.Fields[20].Description = "Redirects specifies whether redirects should be followed by the HTTP Client.\n\nThis can be used in conjunction with `max-redirects` to control the HTTP request redirects." + HTTPRequestDoc.Fields[20].Comments[encoder.LineComment] = "Redirects specifies whether redirects should be followed by the HTTP Client." + HTTPRequestDoc.Fields[21].Name = "pipeline" HTTPRequestDoc.Fields[21].Type = "bool" HTTPRequestDoc.Fields[21].Note = "" - HTTPRequestDoc.Fields[21].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client." - HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests." - HTTPRequestDoc.Fields[22].Name = "race" + HTTPRequestDoc.Fields[21].Description = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining\n\nAll requests must be idempotent (GET/POST). This can be used for race conditions/billions requests." + HTTPRequestDoc.Fields[21].Comments[encoder.LineComment] = "Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining" + HTTPRequestDoc.Fields[22].Name = "unsafe" HTTPRequestDoc.Fields[22].Type = "bool" HTTPRequestDoc.Fields[22].Note = "" - HTTPRequestDoc.Fields[22].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field." - HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)" - HTTPRequestDoc.Fields[23].Name = "req-condition" + HTTPRequestDoc.Fields[22].Description = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests.\n\nThis uses the [rawhttp](https://github.com/projectdiscovery/rawhttp) engine to achieve complete\ncontrol over the request, with no normalization performed by the client." + HTTPRequestDoc.Fields[22].Comments[encoder.LineComment] = "Unsafe specifies whether to use rawhttp engine for sending Non RFC-Compliant requests." + HTTPRequestDoc.Fields[23].Name = "race" HTTPRequestDoc.Fields[23].Type = "bool" HTTPRequestDoc.Fields[23].Note = "" - HTTPRequestDoc.Fields[23].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions." - HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history." - HTTPRequestDoc.Fields[24].Name = "stop-at-first-match" + HTTPRequestDoc.Fields[23].Description = "Race determines if all the request have to be attempted at the same time (Race Condition)\n\nThe actual number of requests that will be sent is determined by the `race_count` field." + HTTPRequestDoc.Fields[23].Comments[encoder.LineComment] = "Race determines if all the request have to be attempted at the same time (Race Condition)" + HTTPRequestDoc.Fields[24].Name = "req-condition" HTTPRequestDoc.Fields[24].Type = "bool" HTTPRequestDoc.Fields[24].Note = "" - HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." - HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." - HTTPRequestDoc.Fields[25].Name = "skip-variables-check" + HTTPRequestDoc.Fields[24].Description = "ReqCondition automatically assigns numbers to requests and preserves their history.\n\nThis allows matching on them later for multi-request conditions." + HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "ReqCondition automatically assigns numbers to requests and preserves their history." + HTTPRequestDoc.Fields[25].Name = "stop-at-first-match" HTTPRequestDoc.Fields[25].Type = "bool" HTTPRequestDoc.Fields[25].Note = "" - HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request" - HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request" - HTTPRequestDoc.Fields[26].Name = "iterate-all" + HTTPRequestDoc.Fields[25].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." + HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." + HTTPRequestDoc.Fields[26].Name = "skip-variables-check" HTTPRequestDoc.Fields[26].Type = "bool" HTTPRequestDoc.Fields[26].Note = "" - HTTPRequestDoc.Fields[26].Description = "IterateAll iterates all the values extracted from internal extractors" - HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors" + HTTPRequestDoc.Fields[26].Description = "SkipVariablesCheck skips the check for unresolved variables in request" + HTTPRequestDoc.Fields[26].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request" + HTTPRequestDoc.Fields[27].Name = "iterate-all" + HTTPRequestDoc.Fields[27].Type = "bool" + HTTPRequestDoc.Fields[27].Note = "" + HTTPRequestDoc.Fields[27].Description = "IterateAll iterates all the values extracted from internal extractors" + HTTPRequestDoc.Fields[27].Comments[encoder.LineComment] = "IterateAll iterates all the values extracted from internal extractors" MATCHERSMatcherDoc.Type = "matchers.Matcher" MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol." @@ -906,6 +924,17 @@ func init() { "PURGE", } + SignatureTypeHolderDoc.Type = "SignatureTypeHolder" + SignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature" + SignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature" + SignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "http.Request", + FieldName: "signature", + }, + } + SignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0) + DNSRequestDoc.Type = "dns.Request" DNSRequestDoc.Comments[encoder.LineComment] = " Request contains a DNS protocol request to be made from a template" DNSRequestDoc.Description = "Request contains a DNS protocol request to be made from a template" @@ -1762,6 +1791,17 @@ func init() { WORKFLOWSMatcherDoc.Fields[1].Note = "" WORKFLOWSMatcherDoc.Fields[1].Description = "Subtemplates are run if the name of matcher matches." WORKFLOWSMatcherDoc.Fields[1].Comments[encoder.LineComment] = "Subtemplates are run if the name of matcher matches." + + HTTPSignatureTypeHolderDoc.Type = "http.SignatureTypeHolder" + HTTPSignatureTypeHolderDoc.Comments[encoder.LineComment] = " SignatureTypeHolder is used to hold internal type of the signature" + HTTPSignatureTypeHolderDoc.Description = "SignatureTypeHolder is used to hold internal type of the signature" + HTTPSignatureTypeHolderDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Template", + FieldName: "signature", + }, + } + HTTPSignatureTypeHolderDoc.Fields = make([]encoder.Doc, 0) } // GetTemplateDoc returns documentation for the file templates_doc.go. @@ -1782,6 +1822,7 @@ func GetTemplateDoc() *encoder.FileDoc { &ExtractorTypeHolderDoc, &GENERATORSAttackTypeHolderDoc, &HTTPMethodTypeHolderDoc, + &SignatureTypeHolderDoc, &DNSRequestDoc, &DNSRequestTypeHolderDoc, &FILERequestDoc, @@ -1797,6 +1838,7 @@ func GetTemplateDoc() *encoder.FileDoc { &WHOISRequestDoc, &WORKFLOWSWorkflowTemplateDoc, &WORKFLOWSMatcherDoc, + &HTTPSignatureTypeHolderDoc, }, } } From 8e62abe6e7f42fec1b49da727c1e5e5746e08cf3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 20 Dec 2021 01:14:23 +0530 Subject: [PATCH 36/41] fix: example reporting config update --- v2/cmd/nuclei/issue-tracker-config.yaml | 90 ++++++++++++------------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index 508446243..367b5ad28 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -1,61 +1,55 @@ -# to specify which severities should be reported #allow-list: -# severity: critical, high -# to specify which severities should be excluded from reporting +# severity: high, critical #deny-list: -# severity: info, low, medium - +# severity: low +# # GitHub contains configuration options for GitHub issue tracker -#GitHub: -# # base-url (optional) is the self-hosted GitHub application url -# base-url: "" +#github: +# # base-url is the optional self-hosted GitHub application url +# base-url: https://localhost:8443/github # # username is the username of the GitHub user -# username: "" -# # owner is the owner name of the repository for issues. -# owner: "" -# # token is the token for GitHub account. -# token: "" -# # project-name is the name of the repository. -# project-name: "" -# # issue-label (optional) is the label of the created issue type -# issue-label: "" -# # severity-as-label (optional) sets the severity as the label of the created issue type -# severity-as-label: false - -# GitLab contains configuration options for GitLab issue tracker -#GitLab: -# # base-url (optional) is the self-hosted GitLab application url -# base-url: "" +# username: test-username +# # owner is the owner name of the repository for issues +# owner: test-owner +# # token is the token for GitHub account +# token: test-token +# # project-name is the name of the repository +# project-name: test-project +# # issue-label is the label of the created issue type +# issue-label: bug +# +# GitLab contains configuration options for gitlab issue tracker +#gitlab: +# # base-url is the optional self-hosted GitLab application url +# base-url: https://localhost:8443/gitlab # # username is the username of the GitLab user -# username: "" -# # token is the token for GitLab account. -# token: "" -# # project-id is the ID of the repository. -# project-id: "" -# # issue-label (optional) is the label of the created issue type -# issue-label: "" -# # severity-as-label (optional) sets the severity as the label of the created issue type -# severity-as-label: false - +# username: test-username +# # token is the token for GitLab account +# token: test-token +# # project-name is the name/id of the project(repository) +# project-name: "1234" +# # issue-label is the label of the created issue type +# issue-label: bug +# # Jira contains configuration options for Jira issue tracker -#Jira: -# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +#jira: +# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true -# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created +# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false -# # URL is the Jira application URL -# url: "" +# # URL is the jira application url +# url: https://localhost/jira # # account-id is the account-id of the Jira user or username in case of on-prem Jira -# account-id: "" +# account-id: test-account-id # # email is the email of the user for Jira instance -# email: "" +# email: test@test.com # # token is the token for Jira instance or password in case of on-prem Jira -# token: "" +# token: test-token # # project-name is the name of the project. -# project-name: "" +# project-name: test-project-name # # issue-type is the name of the created issue type -# issue-type: "" - +# issue-type: bug +# # elasticsearch contains configuration options for elasticsearch exporter #elasticsearch: # # IP for elasticsearch instance @@ -64,11 +58,11 @@ # port: 9200 # # IndexName is the name of the elasticsearch index # index-name: nuclei -# # SSL (optional) enables ssl for elasticsearch connection +# # SSL enables ssl for elasticsearch connection # ssl: false -# # SSLVerification (optional) disables SSL verification for elasticsearch +# # SSLVerification disables SSL verification for elasticsearch # ssl-verification: false # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test +# password: test \ No newline at end of file From d27239e953609d9081f43dc56075b2aa2f294a09 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 20 Dec 2021 12:17:04 +0100 Subject: [PATCH 37/41] Fixing CVE annotate crash --- v2/cmd/cve-annotate/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/cve-annotate/main.go b/v2/cmd/cve-annotate/main.go index c2ccaa765..3e21fe94e 100644 --- a/v2/cmd/cve-annotate/main.go +++ b/v2/cmd/cve-annotate/main.go @@ -71,7 +71,7 @@ func getCVEData(client *nvd.Client, filePath, data string) { cveName := matches[0][1] severityMatches := severityRegex.FindAllStringSubmatch(data, 1) - if len(matches) == 0 { + if len(severityMatches) == 0 { return } severityValue := severityMatches[0][1] From 0a9f890fddf0fbb29320909f7aefecd82d87d682 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 20 Dec 2021 14:11:55 +0100 Subject: [PATCH 38/41] Removing invalid negative paths for windows --- v2/internal/runner/update_test.go | 20 +-------------- v2/internal/runner/update_unix_test.go | 34 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 19 deletions(-) create mode 100644 v2/internal/runner/update_unix_test.go diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index e8153b42e..0983aa71d 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -119,7 +119,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Equal(t, "base.yaml", results.deletions[0], "could not get correct new deletions") } -func TestCalculateTemplateAbsolutePath(t *testing.T) { +func TestCalculateTemplateAbsolutePathPositiveScenario(t *testing.T) { configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") defer os.RemoveAll(configuredTemplateDirectory) @@ -136,24 +136,6 @@ func TestCalculateTemplateAbsolutePath(t *testing.T) { require.False(t, skipFile) } }) - - t.Run("negative scenarios", func(t *testing.T) { - filePathsFromZip := []string{ - "./../nuclei-templates/../cve/test.yaml", - "nuclei-templates/../cve/test.yaml", - "nuclei-templates/cve/../test.yaml", - "nuclei-templates/././../cve/test.yaml", - "nuclei-templates/.././../cve/test.yaml", - "nuclei-templates/.././../cve/../test.yaml", - } - - for _, filePathFromZip := range filePathsFromZip { - calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) - require.Nil(t, err) - require.True(t, skipFile) - require.Equal(t, "", calculatedTemplateAbsPath) - } - }) } func zipFromDirectory(zipPath, directory string) error { diff --git a/v2/internal/runner/update_unix_test.go b/v2/internal/runner/update_unix_test.go new file mode 100644 index 000000000..c9e1a9a2d --- /dev/null +++ b/v2/internal/runner/update_unix_test.go @@ -0,0 +1,34 @@ +//go:build !windows + +package runner + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCalculateTemplateAbsolutePathNegativeScenario(t *testing.T) { + configuredTemplateDirectory := filepath.Join(os.TempDir(), "templates") + defer os.RemoveAll(configuredTemplateDirectory) + + t.Run("negative scenarios", func(t *testing.T) { + filePathsFromZip := []string{ + "./../nuclei-templates/../cve/test.yaml", + "nuclei-templates/../cve/test.yaml", + "nuclei-templates/cve/../test.yaml", + "nuclei-templates/././../cve/test.yaml", + "nuclei-templates/.././../cve/test.yaml", + "nuclei-templates/.././../cve/../test.yaml", + } + + for _, filePathFromZip := range filePathsFromZip { + calculatedTemplateAbsPath, skipFile, err := calculateTemplateAbsolutePath(filePathFromZip, configuredTemplateDirectory) + require.Nil(t, err) + require.True(t, skipFile) + require.Equal(t, "", calculatedTemplateAbsPath) + } + }) +} From 75946f7c0f6b64b479277ddc0b2a789766d93203 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 20 Dec 2021 16:23:36 +0100 Subject: [PATCH 39/41] fixing ip/hostname for windows self-contained templates --- integration_tests/http/self-contained.yaml | 2 +- integration_tests/network/self-contained.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/http/self-contained.yaml b/integration_tests/http/self-contained.yaml index 4ecacbbd9..b3c9aa993 100644 --- a/integration_tests/http/self-contained.yaml +++ b/integration_tests/http/self-contained.yaml @@ -9,7 +9,7 @@ self-contained: true requests: - raw: - | - GET http://localhost:5431/ HTTP/1.1 + GET http://127.0.0.1:5431/ HTTP/1.1 Host: {{Hostname}} matchers: diff --git a/integration_tests/network/self-contained.yaml b/integration_tests/network/self-contained.yaml index fad3e2ac8..711af4f55 100644 --- a/integration_tests/network/self-contained.yaml +++ b/integration_tests/network/self-contained.yaml @@ -8,7 +8,7 @@ info: self-contained: true network: - host: - - "localhost:5431" + - "127.0.0.1:5431" matchers: - type: word From d1944c76b94d16db4a37a2696889b952b4c549d9 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 21 Dec 2021 10:54:25 +0100 Subject: [PATCH 40/41] Improving projectfile http request matching --- v2/pkg/projectfile/project.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/v2/pkg/projectfile/project.go b/v2/pkg/projectfile/project.go index a64d7db1b..84e0a0cb5 100644 --- a/v2/pkg/projectfile/project.go +++ b/v2/pkg/projectfile/project.go @@ -1,12 +1,20 @@ package projectfile import ( - "fmt" "net/http" + "regexp" + + "github.com/pkg/errors" "github.com/projectdiscovery/hmap/store/hybrid" ) +var ( + ErrNotFound = errors.New("not found") + regexUserAgent = regexp.MustCompile(`(?mi)\r\nUser-Agent: .+\r\n`) + regexDefaultInteract = regexp.MustCompile(`(?mi)[a-zA-Z1-9%.]+interact.sh`) +) + type Options struct { Path string Cleanup bool @@ -31,15 +39,22 @@ func New(options *Options) (*ProjectFile, error) { return &p, nil } +func (pf *ProjectFile) cleanupData(data []byte) []byte { + // ignore all user agents + data = regexUserAgent.ReplaceAll(data, []byte("\r\n")) + // ignore interact markers + return regexDefaultInteract.ReplaceAll(data, []byte("")) +} + func (pf *ProjectFile) Get(req []byte) (*http.Response, error) { - reqHash, err := hash(req) + reqHash, err := hash(pf.cleanupData(req)) if err != nil { return nil, err } data, ok := pf.hm.Get(reqHash) if !ok { - return nil, fmt.Errorf("not found") + return nil, ErrNotFound } var httpRecord HTTPRecord @@ -52,7 +67,7 @@ func (pf *ProjectFile) Get(req []byte) (*http.Response, error) { } func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error { - reqHash, err := hash(req) + reqHash, err := hash(pf.cleanupData(req)) if err != nil { return err } From e6aabffc2287b7158728131b86933aab92fd435f Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 21 Dec 2021 19:02:09 +0530 Subject: [PATCH 41/41] add missing custom rdap server assign if present in the template --- v2/pkg/protocols/whois/whois.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 6da0305f5..5f46dc433 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -85,6 +85,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous query := replacer.Replace(request.Query, variables) // build an rdap request rdapReq := rdap.NewAutoRequest(query) + rdapReq.Server = request.parsedServerURL res, err := request.client.Do(rdapReq) if err != nil { return errors.Wrap(err, "could not make whois request")