nuclei/v2/pkg/external/customtemplates/templates_provider.go

86 lines
2.7 KiB
Go
Raw Normal View History

package customtemplates
import (
"context"
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
const (
CustomGithubTemplateDirectory = "github"
CustomS3TemplateDirectory = "s3"
CustomAzureTemplateDirectory = "azure"
)
type Provider interface {
Download(location string, ctx context.Context)
Update(location string, ctx context.Context)
}
// ParseCustomTemplates function reads the options.GithubTemplateRepo list,
// Checks the given repos are valid or not and stores them into runner.CustomTemplates
func ParseCustomTemplates(options *types.Options) []Provider {
cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K <jaideep@one2n.in> * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com> Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K <jaideep@one2n.in>
2022-12-21 22:48:43 +05:30
if options.Cloud {
return nil
}
var customTemplates []Provider
gitHubClient := getGHClientIncognito()
for _, repoName := range options.GithubTemplateRepo {
owner, repo, err := getOwnerAndRepo(repoName)
if err != nil {
cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K <jaideep@one2n.in> * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com> Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K <jaideep@one2n.in>
2022-12-21 22:48:43 +05:30
gologger.Error().Msgf("%s", err)
continue
}
githubRepo, err := getGithubRepo(gitHubClient, owner, repo, options.GithubToken)
if err != nil {
cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K <jaideep@one2n.in> * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com> Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K <jaideep@one2n.in>
2022-12-21 22:48:43 +05:30
gologger.Error().Msgf("%s", err)
continue
}
customTemplateRepo := &customTemplateGithubRepo{
owner: owner,
reponame: repo,
gitCloneURL: githubRepo.GetCloneURL(),
githubToken: options.GithubToken,
}
customTemplates = append(customTemplates, customTemplateRepo)
}
if options.AwsBucketName != "" {
s3c, err := getS3Client(context.TODO(), options.AwsAccessKey, options.AwsSecretKey, options.AwsRegion)
if err != nil {
gologger.Error().Msgf("error downloading s3 bucket %s %s", options.AwsBucketName, err)
return customTemplates
}
ctBucket := &customTemplateS3Bucket{
bucketName: options.AwsBucketName,
s3Client: s3c,
}
if strings.Contains(options.AwsBucketName, "/") {
bPath := strings.SplitN(options.AwsBucketName, "/", 2)
ctBucket.bucketName = bPath[0]
ctBucket.prefix = bPath[1]
}
customTemplates = append(customTemplates, ctBucket)
}
if options.AzureContainerName != "" {
// Establish a connection to Azure and build a client object with which to download templates from Azure Blob Storage
azClient, err := getAzureBlobClient(options.AzureTenantID, options.AzureClientID, options.AzureClientSecret, options.AzureServiceURL)
if err != nil {
gologger.Error().Msgf("Error establishing Azure Blob client for %s %s", options.AzureContainerName, err)
return customTemplates
}
// Create a new Azure Blob Storage container object
azTemplateContainer := &customTemplateAzureBlob{
azureBlobClient: azClient,
containerName: options.AzureContainerName,
}
// Add the Azure Blob Storage container object to the list of custom templates
customTemplates = append(customTemplates, azTemplateContainer)
}
return customTemplates
}