mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2025-12-18 05:05:23 +00:00
Add s3 bucket template provider
- Refactor the custom github template code - add interface for template provider
This commit is contained in:
shubhamrasal
parent
2aaf2a2158
commit
146e328c6b
@ -269,6 +269,10 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.StringVarEnv(&options.GithubToken, "github-token", "gt", "", "GITHUB_TOKEN", "github token to download public/private templates (GITHUB_TOKEN)"),
|
||||
flagSet.StringSliceVarP(&options.GithubTemplateRepo, "github-template-repo", "gtr", []string{}, "github template repository to download / update (GITHUB_TEMPLATE_REPO)", goflags.FileCommaSeparatedStringSliceOptions),
|
||||
flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"),
|
||||
flagSet.StringVarEnv(&options.AwsAccessKey, "aws-access-key", "aak", "", "AWS_ACCESS_KEY", "aws access key to download template from bucket (AWS_ACCESS_KEY)"),
|
||||
flagSet.StringVarEnv(&options.AwsSecretKey, "aws-secret-key", "ask", "", "AWS_SECRET_KEY", "aws secret key to download template from bucket (AWS_SECRET_KEY)"),
|
||||
flagSet.StringVarEnv(&options.AwsRegion, "aws-region-name", "arg", "", "AWS_REGION", "aws s3 bucket region name to download template(AWS_REGION)"),
|
||||
flagSet.StringVarEnv(&options.AwsBucketName, "aws-bucket-name", "abn", "", "AWS_BUCKET_NAME", "aws s3 bucket name to download template(AWS_BUCKET_NAME)"),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("stats", "Statistics",
|
||||
|
||||
19
v2/go.mod
19
v2/go.mod
@ -65,6 +65,10 @@ require (
|
||||
github.com/DataDog/gostackparse v0.6.0
|
||||
github.com/antchfx/xmlquery v1.3.12
|
||||
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.10
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.1
|
||||
github.com/docker/go-units v0.5.0
|
||||
github.com/fatih/structs v1.1.0
|
||||
github.com/go-git/go-git/v5 v5.4.2
|
||||
@ -200,6 +204,21 @@ require (
|
||||
github.com/Microsoft/go-winio v0.4.16 // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
|
||||
github.com/acomagu/bufpipe v1.0.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/emirpasic/gods v1.12.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.3.1 // indirect
|
||||
|
||||
38
v2/go.sum
38
v2/go.sum
@ -116,6 +116,44 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W
|
||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.44.129 h1:yld8Rc8OCahLtenY1mnve4w1jVeBu/rSiscGzodaDOs=
|
||||
github.com/aws/aws-sdk-go v1.44.129/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 h1:RKci2D7tMwpvGpDNZnGQw9wk6v7o/xSwFcUAuNPoB8k=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9/go.mod h1:vCmV1q1VK8eoQJ5+aYE7PkK1K6v41qJ5pJdK3ggCDvg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.10/go.mod h1:/4np+UiJJKpWHN7Q+LZvqXYgyjgeXm5+lLfDI6TPZao=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.23 h1:LctvcJMIb8pxvk5hQhChpCu0WlU6oKQmcYb1HA4IZSA=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.23/go.mod h1:0awX9iRr/+UO7OwRQFpV1hNtXxOVuehpjVEzrIAYNcA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37 h1:e1VtTBo+cLNjres0wTlMkmwCGGRjDEkkrz3frxxcaCs=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37/go.mod h1:kdAV1UMnCkyG6tZJUC4mHbPoRjPA3dIK0L8mnsHERiM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16 h1:2EXB7dtGwRYIN3XQ9qwIW504DVbKIw3r89xQnonGdsQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.16/go.mod h1:XH+3h395e3WVdd6T2Z3mPxuI+x/HVtdqVOREkTiyubs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 h1:dpiPHgmFstgkLG07KaYAewvuptq5kvo52xn7tVSrtrQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20 h1:KSvtm1+fPXE0swe9GPjc6msyrdTT0LB/BP8eLugL1FI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.20/go.mod h1:Mp4XI/CkWGD79AQxZ5lIFlgvC0A+gl+4BmyG1F+SfNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 h1:piDBAaWkaxkkVV3xJJbTehXCZRXYs49kvpi/LG6LR2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19/go.mod h1:BmQWRVkLTmyNzYPFAZgon53qKLWBNSvonugD1MrSWUs=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.1 h1:/EMdFPW/Ppieh0WUtQf1+qCGNLdsq5UWUyevBQ6vMVc=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.1/go.mod h1:/NHbqPRiwxSPVOB2Xr+StDEH+GWV/64WwnUjv4KYzV0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 h1:KRAix/KHvjGODaHAMXnxRk9t0D+4IJVUuS/uwXxngXk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.1/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4=
|
||||
github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk=
|
||||
github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
|
||||
264
v2/internal/runner/custom_templates.go
Normal file
264
v2/internal/runner/custom_templates.go
Normal file
@ -0,0 +1,264 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/google/go-github/github"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
|
||||
)
|
||||
|
||||
type customTemplateProvider interface {
|
||||
Download(location string, ctx context.Context)
|
||||
Update(location string, ctx context.Context)
|
||||
}
|
||||
|
||||
type customTemplateGithubRepo struct {
|
||||
owner string
|
||||
reponame string
|
||||
gitCloneURL string
|
||||
githubToken string
|
||||
}
|
||||
|
||||
type customTemplateS3Bucket struct {
|
||||
bucketName string
|
||||
prefix string
|
||||
acccessKey string
|
||||
secretKey string
|
||||
region string
|
||||
}
|
||||
|
||||
// parseCustomTemplates function reads the options.GithubTemplateRepo list,
|
||||
// Checks the given repos are valid or not and stores them into runner.CustomTemplates
|
||||
func (r *Runner) parseCustomTemplates() *[]customTemplateProvider {
|
||||
var customTemplates []customTemplateProvider
|
||||
gitHubClient := getGHClientIncognito()
|
||||
|
||||
for _, repoName := range r.options.GithubTemplateRepo {
|
||||
owner, repo, err := getOwnerAndRepo(repoName)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
continue
|
||||
}
|
||||
githubRepo, err := getGithubRepo(gitHubClient, owner, repo)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
continue
|
||||
}
|
||||
customTemplateRepo := &customTemplateGithubRepo{
|
||||
owner: owner,
|
||||
reponame: repo,
|
||||
gitCloneURL: githubRepo.GetCloneURL(),
|
||||
githubToken: r.options.GithubToken,
|
||||
}
|
||||
customTemplates = append(customTemplates, customTemplateRepo)
|
||||
}
|
||||
if r.options.AwsBucketName != "" {
|
||||
ctBucket := &customTemplateS3Bucket{
|
||||
bucketName: r.options.AwsBucketName,
|
||||
acccessKey: r.options.AwsAccessKey,
|
||||
secretKey: r.options.AwsSecretKey,
|
||||
region: r.options.AwsRegion,
|
||||
}
|
||||
if strings.Contains(r.options.AwsBucketName, "/") {
|
||||
bPath := strings.SplitN(r.options.AwsBucketName, "/", 2)
|
||||
ctBucket.bucketName = bPath[0]
|
||||
ctBucket.prefix = bPath[1]
|
||||
}
|
||||
customTemplates = append(customTemplates, ctBucket)
|
||||
}
|
||||
return &customTemplates
|
||||
}
|
||||
|
||||
// This function download the custom github template repository
|
||||
func (customTemplate *customTemplateGithubRepo) Download(location string, ctx context.Context) {
|
||||
downloadPath := filepath.Join(location, customGithubTemplateDirectory)
|
||||
clonePath := customTemplate.getLocalRepoClonePath(downloadPath)
|
||||
|
||||
if !fileutil.FolderExists(clonePath) {
|
||||
err := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
} else {
|
||||
gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (customTemplate *customTemplateGithubRepo) Update(location string, ctx context.Context) {
|
||||
downloadPath := filepath.Join(location, customGithubTemplateDirectory)
|
||||
clonePath := customTemplate.getLocalRepoClonePath(downloadPath)
|
||||
|
||||
// If folder does not exits then clone/download the repo
|
||||
if !fileutil.FolderExists(clonePath) {
|
||||
customTemplate.Download(location, ctx)
|
||||
return
|
||||
}
|
||||
err := customTemplate.pullChanges(clonePath, customTemplate.githubToken)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
} else {
|
||||
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
|
||||
}
|
||||
}
|
||||
|
||||
// getOwnerAndRepo returns the owner, repo, err from the given string
|
||||
// eg. it takes input projectdiscovery/nuclei-templates and
|
||||
// returns owner=> projectdiscovery , repo => nuclei-templates
|
||||
func getOwnerAndRepo(reponame string) (owner string, repo string, err error) {
|
||||
s := strings.Split(reponame, "/")
|
||||
if len(s) != 2 {
|
||||
err = errors.Errorf("wrong Repo name: %s", reponame)
|
||||
return
|
||||
}
|
||||
owner = s[0]
|
||||
repo = s[1]
|
||||
return
|
||||
}
|
||||
|
||||
// returns *github.Repository if passed github repo name
|
||||
func getGithubRepo(gitHubClient *github.Client, repoOwner, repoName string) (*github.Repository, error) {
|
||||
var retried bool
|
||||
getRepo:
|
||||
repo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName)
|
||||
if err != nil {
|
||||
// retry with authentication
|
||||
if gitHubClient = getGHClientWithToken(); gitHubClient != nil && !retried {
|
||||
retried = true
|
||||
goto getRepo
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if repo == nil {
|
||||
return nil, errors.Errorf("problem getting repository: %s/%s", repoOwner, repoName)
|
||||
}
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
// download the git repo to given path
|
||||
func (ctr *customTemplateGithubRepo) cloneRepo(clonePath, githubToken string) error {
|
||||
r, err := git.PlainClone(clonePath, false, &git.CloneOptions{
|
||||
URL: ctr.gitCloneURL,
|
||||
Auth: getAuth(ctr.owner, githubToken),
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
|
||||
}
|
||||
// Add the user as well in the config. By default user is not set
|
||||
config, _ := r.Storer.Config()
|
||||
config.User.Name = ctr.owner
|
||||
return r.SetConfig(config)
|
||||
}
|
||||
|
||||
// performs the git pull on given repo
|
||||
func (ctr *customTemplateGithubRepo) pullChanges(repoPath, githubToken string) error {
|
||||
r, err := git.PlainOpen(repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := r.Worktree()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = w.Pull(&git.PullOptions{RemoteName: "origin", Auth: getAuth(ctr.owner, githubToken)})
|
||||
if err != nil {
|
||||
return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getLocalRepoClonePath returns the clone path.
|
||||
// if same name repo directory exists from another owner then it appends the owner then and returns the path
|
||||
// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep
|
||||
func (ctr *customTemplateGithubRepo) getLocalRepoClonePath(downloadPath string) string {
|
||||
if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) {
|
||||
return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
|
||||
}
|
||||
return filepath.Join(downloadPath, ctr.reponame)
|
||||
}
|
||||
|
||||
// isRepoDirExists take the path and checks if the same repo or not
|
||||
func (ctr *customTemplateGithubRepo) isRepoDirExists(repoPath string) bool {
|
||||
r, _ := git.PlainOpen(repoPath)
|
||||
local, _ := r.Config()
|
||||
return local.User.Name == ctr.owner // repo already cloned no need to rename and clone
|
||||
}
|
||||
|
||||
// returns the auth object with username and github token as password
|
||||
func getAuth(username, password string) *http.BasicAuth {
|
||||
if username != "" && password != "" {
|
||||
return &http.BasicAuth{Username: username, Password: password}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// download custom templates from s3 bucket
|
||||
func (bk *customTemplateS3Bucket) Download(location string, ctx context.Context) {
|
||||
downloadPath := filepath.Join(location, customS3TemplateDirectory, bk.bucketName)
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(bk.acccessKey, bk.secretKey, "")), config.WithRegion(bk.region))
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("error downloading aws bucket %s %s", bk.bucketName, err)
|
||||
return
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(cfg)
|
||||
manager := manager.NewDownloader(client)
|
||||
|
||||
paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
|
||||
Bucket: &bk.bucketName,
|
||||
Prefix: &bk.prefix,
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("error downloading aws bucket %s %s", bk.bucketName, err)
|
||||
return
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
if err := downloadToFile(manager, downloadPath, bk.bucketName, aws.ToString(obj.Key)); err != nil {
|
||||
gologger.Error().Msgf("error downloading aws bucket %s %s", bk.bucketName, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
gologger.Info().Msgf("AWS bucket %s successfully cloned successfully at %s", bk.bucketName, downloadPath)
|
||||
}
|
||||
|
||||
// download custom templates from s3 bucket
|
||||
func (bk *customTemplateS3Bucket) Update(location string, ctx context.Context) {
|
||||
bk.Download(location, ctx)
|
||||
}
|
||||
|
||||
func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key string) error {
|
||||
// Create the directories in the path
|
||||
file := filepath.Join(targetDirectory, key)
|
||||
if err := os.MkdirAll(filepath.Dir(file), 0775); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set up the local file
|
||||
fd, err := os.Create(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
// Download the file using the AWS SDK for Go
|
||||
_, err = downloader.Download(context.TODO(), fd, &s3.GetObjectInput{Bucket: &bucket, Key: &key})
|
||||
|
||||
return err
|
||||
}
|
||||
@ -1,165 +0,0 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/google/go-github/github"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
|
||||
)
|
||||
|
||||
type customTemplateRepo struct {
|
||||
owner string
|
||||
reponame string
|
||||
gitCloneURL string
|
||||
}
|
||||
|
||||
// This function download the custom template repository
|
||||
// scenario 1: -gtr custom-template.txt flag has passed => Only download the repos. Do not update
|
||||
// scenario 2: -gtr custom-template.txt -tup github => Update the repo(git pull) and download if any new repo
|
||||
// Reason to add update and download logic in single function is scenario 2
|
||||
func (r *Runner) downloadCustomTemplates(ctx context.Context) {
|
||||
downloadPath := filepath.Join(r.templatesConfig.TemplatesDirectory, customTemplateType)
|
||||
|
||||
for _, customTemplate := range r.customTemplates {
|
||||
clonePath := customTemplate.getLocalRepoClonePath(downloadPath)
|
||||
|
||||
if !fileutil.FolderExists(clonePath) {
|
||||
err := customTemplate.cloneRepo(clonePath, r.options.GithubToken)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
} else {
|
||||
gologger.Info().Msgf("Repo %s/%s cloned successfully at %s", customTemplate.owner, customTemplate.reponame, clonePath)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if r.options.UpdateTemplates {
|
||||
err := customTemplate.pullChanges(clonePath, r.options.GithubToken)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
} else {
|
||||
gologger.Info().Msgf("Repo %s/%s successfully pulled the changes.\n", customTemplate.owner, customTemplate.reponame)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseCustomTemplates function reads the options.GithubTemplateRepo list,
|
||||
// Checks the given repos are valid or not and stores them into runner.CustomTemplates
|
||||
func (r *Runner) parseCustomTemplates() {
|
||||
gitHubClient := getGHClientIncognito()
|
||||
|
||||
for _, repoName := range r.options.GithubTemplateRepo {
|
||||
owner, repo, err := getOwnerAndRepo(repoName)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
continue
|
||||
}
|
||||
githubRepo, err := getGithubRepo(gitHubClient, owner, repo)
|
||||
if err != nil {
|
||||
gologger.Info().Msgf("%s", err)
|
||||
continue
|
||||
}
|
||||
customTemplateRepo := &customTemplateRepo{
|
||||
owner: owner,
|
||||
reponame: repo,
|
||||
gitCloneURL: githubRepo.GetCloneURL(),
|
||||
}
|
||||
r.customTemplates = append(r.customTemplates, *customTemplateRepo)
|
||||
}
|
||||
}
|
||||
|
||||
// getOwnerAndRepo returns the owner, repo, err from the given string
|
||||
// eg. it takes input projectdiscovery/nuclei-templates and
|
||||
// returns owner=> projectdiscovery , repo => nuclei-templates
|
||||
func getOwnerAndRepo(reponame string) (owner string, repo string, err error) {
|
||||
s := strings.Split(reponame, "/")
|
||||
if len(s) != 2 {
|
||||
err = errors.Errorf("wrong Repo name: %s", reponame)
|
||||
return
|
||||
}
|
||||
owner = s[0]
|
||||
repo = s[1]
|
||||
return
|
||||
}
|
||||
|
||||
// returns *github.Repository if passed github repo name
|
||||
func getGithubRepo(gitHubClient *github.Client, repoOwner, repoName string) (*github.Repository, error) {
|
||||
var retried bool
|
||||
getRepo:
|
||||
repo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName)
|
||||
if err != nil {
|
||||
// retry with authentication
|
||||
if gitHubClient = getGHClientWithToken(); gitHubClient != nil && !retried {
|
||||
retried = true
|
||||
goto getRepo
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if repo == nil {
|
||||
return nil, errors.Errorf("problem getting repository: %s/%s", repoOwner, repoName)
|
||||
}
|
||||
return repo, nil
|
||||
}
|
||||
|
||||
// download the git repo to given path
|
||||
func (ctr *customTemplateRepo) cloneRepo(clonePath, githubToken string) error {
|
||||
r, err := git.PlainClone(clonePath, false, &git.CloneOptions{
|
||||
URL: ctr.gitCloneURL,
|
||||
Auth: getAuth(ctr.owner, githubToken),
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
|
||||
}
|
||||
// Add the user as well in the config. By default user is not set
|
||||
config, _ := r.Storer.Config()
|
||||
config.User.Name = ctr.owner
|
||||
return r.SetConfig(config)
|
||||
}
|
||||
|
||||
// performs the git pull on given repo
|
||||
func (ctr *customTemplateRepo) pullChanges(repoPath, githubToken string) error {
|
||||
r, err := git.PlainOpen(repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := r.Worktree()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = w.Pull(&git.PullOptions{RemoteName: "origin", Auth: getAuth(ctr.owner, githubToken)})
|
||||
if err != nil {
|
||||
return errors.Errorf("%s/%s: %s", ctr.owner, ctr.reponame, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getLocalRepoClonePath returns the clone path.
|
||||
// if same name repo directory exists from another owner then it appends the owner then and returns the path
|
||||
// eg. for nuclei-templates directory exists for projectdiscovery owner, then for ehsandeep/nuclei-templates it will return nuclei-templates-ehsandeep
|
||||
func (ctr *customTemplateRepo) getLocalRepoClonePath(downloadPath string) string {
|
||||
if fileutil.FolderExists(filepath.Join(downloadPath, ctr.reponame)) && !ctr.isRepoDirExists(filepath.Join(downloadPath, ctr.reponame)) {
|
||||
return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
|
||||
}
|
||||
return filepath.Join(downloadPath, ctr.reponame)
|
||||
}
|
||||
|
||||
// isRepoDirExists take the path and checks if the same repo or not
|
||||
func (ctr *customTemplateRepo) isRepoDirExists(repoPath string) bool {
|
||||
r, _ := git.PlainOpen(repoPath)
|
||||
local, _ := r.Config()
|
||||
return local.User.Name == ctr.owner // repo already cloned no need to rename and clone
|
||||
}
|
||||
|
||||
// returns the auth object with username and github token as password
|
||||
func getAuth(username, password string) *http.BasicAuth {
|
||||
if username != "" && password != "" {
|
||||
return &http.BasicAuth{Username: username, Password: password}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -23,9 +23,11 @@ func TestDownloadCustomTemplates(t *testing.T) {
|
||||
options.GithubTemplateRepo = []string{"projectdiscovery/nuclei-templates", "ehsandeep/nuclei-templates"}
|
||||
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}, options: options}
|
||||
|
||||
r.parseCustomTemplates()
|
||||
r.customTemplates = r.parseCustomTemplates()
|
||||
|
||||
r.downloadCustomTemplates(context.Background())
|
||||
for _, ct := range r.customTemplates {
|
||||
ct.Download(r.templatesConfig.TemplatesDirectory, context.Background())
|
||||
}
|
||||
|
||||
require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates"), "cloned directory does not exists")
|
||||
require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists")
|
||||
|
||||
@ -95,6 +95,11 @@ func ParseOptions(options *types.Options) {
|
||||
if err != nil {
|
||||
gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
|
||||
}
|
||||
|
||||
// Set Github token in env variable. runner.getGHClientWithToken() reads token from env
|
||||
if options.GithubToken != "" && os.Getenv("GITHUB_TOKEN") != options.GithubToken {
|
||||
os.Setenv("GITHUB_TOKEN", options.GithubToken)
|
||||
}
|
||||
}
|
||||
|
||||
// validateOptions validates the configuration options passed
|
||||
|
||||
@ -72,7 +72,7 @@ type Runner struct {
|
||||
hostErrors hosterrorscache.CacheInterface
|
||||
resumeCfg *types.ResumeCfg
|
||||
pprofServer *http.Server
|
||||
customTemplates []customTemplateRepo
|
||||
customTemplates *[]customTemplateProvider
|
||||
cloudClient *nucleicloud.Client
|
||||
}
|
||||
|
||||
@ -107,7 +107,7 @@ func New(options *types.Options) (*Runner, error) {
|
||||
parsers.NoStrictSyntax = options.NoStrictSyntax
|
||||
|
||||
// parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos
|
||||
runner.parseCustomTemplates()
|
||||
runner.customTemplates = runner.parseCustomTemplates()
|
||||
|
||||
if err := runner.updateTemplates(); err != nil {
|
||||
gologger.Error().Msgf("Could not update templates: %s\n", err)
|
||||
|
||||
@ -37,11 +37,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
userName = "projectdiscovery"
|
||||
repoName = "nuclei-templates"
|
||||
nucleiIgnoreFile = ".nuclei-ignore"
|
||||
nucleiConfigFilename = ".templates-config.json"
|
||||
customTemplateType = "github"
|
||||
userName = "projectdiscovery"
|
||||
repoName = "nuclei-templates"
|
||||
nucleiIgnoreFile = ".nuclei-ignore"
|
||||
nucleiConfigFilename = ".templates-config.json"
|
||||
customGithubTemplateDirectory = "github"
|
||||
customS3TemplateDirectory = "s3"
|
||||
)
|
||||
|
||||
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
|
||||
@ -104,7 +105,13 @@ func (r *Runner) updateTemplates() error { // TODO this method does more than ju
|
||||
}
|
||||
|
||||
// download | update the custom templates repos
|
||||
r.downloadCustomTemplates(ctx)
|
||||
for _, ct := range *r.customTemplates {
|
||||
if r.options.UpdateTemplates {
|
||||
ct.Update(r.templatesConfig.TemplatesDirectory, ctx)
|
||||
} else {
|
||||
ct.Download(r.templatesConfig.TemplatesDirectory, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
latestVersion, currentVersion, err := getVersions(r)
|
||||
if err != nil {
|
||||
@ -168,7 +175,9 @@ func (r *Runner) freshTemplateInstallation(configDir string, ctx context.Context
|
||||
gologger.Info().Msgf("Successfully downloaded nuclei-templates (v%s) to %s. GoodLuck!\n", version.String(), r.templatesConfig.TemplatesDirectory)
|
||||
|
||||
// case where -gtr flag is passed for the first time installation
|
||||
r.downloadCustomTemplates(ctx)
|
||||
for _, ct := range *r.customTemplates {
|
||||
ct.Download(r.templatesConfig.TemplatesDirectory, ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -276,6 +276,14 @@ type Options struct {
|
||||
GithubToken string
|
||||
// GithubTemplateRepo is the list of custom public/private templates github repos
|
||||
GithubTemplateRepo goflags.StringSlice
|
||||
// AWS access key for downloading templates from s3 bucket
|
||||
AwsAccessKey string
|
||||
// AWS secret key for downloading templates from s3 bucket
|
||||
AwsSecretKey string
|
||||
// AWS bucket name for downloading templates from s3 bucket
|
||||
AwsBucketName string
|
||||
// AWS Region name where aws s3 bucket is located
|
||||
AwsRegion string
|
||||
|
||||
ConfigPath string // Used by healthcheck
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user