External/nuclei
1
0
Fork 0
mirror of https://github.com/projectdiscovery/nuclei.git synced 2025-12-17 15:55:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tag based struct validation (#1256)

Browse Source 
* Added tag based struct validation
This commit is contained in:
Sajad 2021-11-20 13:25:27 +05:30 committed by GitHub
parent b8246ab8e5
commit f74ff3fc49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 94 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
integration_tests/test-issue-tracker-config1.yaml
View File

@ -26,8 +26,8 @@ gitlab:
username: test-username
# token is the token for gitlab account.
token: test-token
# project-id is the ID of the repository.
project-id: 1234
# project-name is the name/id of the project(repository).
project-name: "1234"
# issue-label is the label of the created issue type
issue-label: bug

4
integration_tests/test-issue-tracker-config2.yaml
View File

@ -28,8 +28,8 @@ gitlab:
username: test-username
# token is the token for gitlab account.
token: test-token
# project-id is the ID of the repository.
project-id: 1234
# project-name is the name/id of the project(repository).
project-name: "1234"
# issue-label is the label of the created issue type
issue-label: bug

5
v2/go.mod
View File

@ -12,6 +12,7 @@ require (
github.com/blang/semver v3.5.1+incompatible
github.com/bluele/gcache v0.0.2
github.com/corpix/uarand v0.1.1
github.com/go-playground/validator/v10 v10.9.0
github.com/go-rod/rod v0.101.8
github.com/gobwas/ws v1.1.0
github.com/google/go-github v17.0.0+incompatible
@ -82,6 +83,8 @@ require (
github.com/eggsampler/acme/v3 v3.2.1 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/go-ole/go-ole v1.2.5 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
@ -100,6 +103,7 @@ require (
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/mattn/go-isatty v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@ -119,6 +123,7 @@ require (
github.com/ysmood/goob v0.3.0 // indirect
github.com/zclconf/go-cty v1.8.4 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/appengine v1.6.7 // indirect

14
v2/go.sum
View File

@ -230,6 +230,14 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A=
github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI=
github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU=
@ -456,6 +464,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
@ -841,6 +851,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -1014,8 +1026,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

15
v2/internal/runner/options.go
View File

@ -2,11 +2,13 @@ package runner
import (
"bufio"
"errors"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/go-playground/validator/v10"
"github.com/projectdiscovery/fileutil"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
@ -77,6 +79,17 @@ func hasStdin() bool {
// validateOptions validates the configuration options passed
func validateOptions(options *types.Options) error {
validate := validator.New()
if err := validate.Struct(options); err != nil {
if _, ok := err.(*validator.InvalidValidationError); ok {
return err
}
errs := []string{}
for _, err := range err.(validator.ValidationErrors) {
errs = append(errs, err.Namespace()+": "+err.Tag())
}
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
}
if options.Verbose && options.Silent {
return errors.New("both verbose and silent mode specified")
}

6
v2/internal/runner/runner.go
View File

@ -10,7 +10,6 @@ import (
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"go.uber.org/ratelimit"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
@ -35,6 +34,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
)
// Runner is a client for running the enumeration process.
@ -180,9 +180,9 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
reportingOptions = &reporting.Options{}
if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
if err := yamlwrapper.DecodeAndValidate(file, reportingOptions); err != nil {
file.Close()
return nil, errors.Wrap(parseErr, "could not parse reporting config file")
return nil, errors.Wrap(err, "could not parse reporting config file")
}
file.Close()
}

40
v2/pkg/reporting/exporters/es/elasticsearch.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
"encoding/base64"
@ -20,19 +19,19 @@ import (
// Options contains necessary options required for elasticsearch communicaiton
type Options struct {
// IP for elasticsearch instance
IP string `yaml:"ip"`
IP string `yaml:"ip" validate:"required,ip"`
// Port is the port of elasticsearch instance
Port int `yaml:"port"`
Port int `yaml:"port" validate:"required,gte=0,lte=65535"`
// SSL (optional) enables ssl for elasticsearch connection
SSL bool `yaml:"ssl"`
// SSLVerification (optional) disables SSL verification for elasticsearch
SSLVerification bool `yaml:"ssl-verification"`
// Username for the elasticsearch instance
Username string `yaml:"username"`
Username string `yaml:"username" validate:"required"`
// Password is the password for elasticsearch instance
Password string `yaml:"password"`
Password string `yaml:"password" validate:"required"`
// IndexName is the name of the elasticsearch index
IndexName string `yaml:"index-name"`
IndexName string `yaml:"index-name" validate:"required"`
}
type data struct {
@ -50,10 +49,6 @@ type Exporter struct {
// New creates and returns a new exporter for elasticsearch
func New(option *Options) (*Exporter, error) {
var ei *Exporter
err := validateOptions(option)
if err != nil {
return nil, err
}
client := &http.Client{
Timeout: 5 * time.Second,
@ -86,31 +81,6 @@ func New(option *Options) (*Exporter, error) {
return ei, nil
}
func validateOptions(options *Options) error {
errs := []string{}
if options.IP == "" {
errs = append(errs, "IP")
}
if options.Port == 0 {
errs = append(errs, "Port")
}
if options.Username == "" {
errs = append(errs, "Username")
}
if options.Password == "" {
errs = append(errs, "Password")
}
if options.IndexName == "" {
errs = append(errs, "IndexName")
}
if len(errs) > 0 {
return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
}
return nil
}
// Export exports a passed result event to elasticsearch
func (i *Exporter) Export(event *output.ResultEvent) error {
// creating a request

36
v2/pkg/reporting/trackers/github/github.go
View File

@ -24,15 +24,15 @@ type Integration struct {
// Options contains the configuration options for github issue tracker client
type Options struct {
// BaseURL (optional) is the self-hosted github application url
BaseURL string `yaml:"base-url"`
BaseURL string `yaml:"base-url" validate:"omitempty,url"`
// Username is the username of the github user
Username string `yaml:"username"`
Username string `yaml:"username" validate:"required"`
// Owner (manadatory) is the owner name of the repository for issues.
Owner string `yaml:"owner"`
Owner string `yaml:"owner" validate:"required"`
// Token is the token for github account.
Token string `yaml:"token"`
Token string `yaml:"token" validate:"required"`
// ProjectName is the name of the repository.
ProjectName string `yaml:"project-name"`
ProjectName string `yaml:"project-name" validate:"required"`
// IssueLabel (optional) is the label of the created issue type
IssueLabel string `yaml:"issue-label"`
// SeverityAsLabel (optional) sends the severity as the label of the created
@ -42,10 +42,6 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
err := validateOptions(options)
if err != nil {
return nil, err
}
ctx := context.Background()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: options.Token},
@ -66,28 +62,6 @@ func New(options *Options) (*Integration, error) {
return &Integration{client: client, options: options}, nil
}
func validateOptions(options *Options) error {
errs := []string{}
if options.Username == "" {
errs = append(errs, "Username")
}
if options.Owner == "" {
errs = append(errs, "Owner")
}
if options.Token == "" {
errs = append(errs, "Token")
}
if options.ProjectName == "" {
errs = append(errs, "ProjectName")
}
if len(errs) > 0 {
return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
}
return nil
}
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)

34
v2/pkg/reporting/trackers/gitlab/gitlab.go
View File

@ -2,9 +2,6 @@ package gitlab
import (
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
@ -21,13 +18,13 @@ type Integration struct {
// Options contains the configuration options for gitlab issue tracker client
type Options struct {
// BaseURL (optional) is the self-hosted gitlab application url
BaseURL string `yaml:"base-url"`
BaseURL string `yaml:"base-url" validate:"omitempty,url"`
// Username is the username of the gitlab user
Username string `yaml:"username"`
Username string `yaml:"username" validate:"required"`
// Token is the token for gitlab account.
Token string `yaml:"token"`
Token string `yaml:"token" validate:"required"`
// ProjectName is the name of the repository.
ProjectName string `yaml:"project-name"`
ProjectName string `yaml:"project-name" validate:"required"`
// IssueLabel is the label of the created issue type
IssueLabel string `yaml:"issue-label"`
// SeverityAsLabel (optional) sends the severity as the label of the created
@ -37,10 +34,6 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
err := validateOptions(options)
if err != nil {
return nil, err
}
gitlabOpts := []gitlab.ClientOptionFunc{}
if options.BaseURL != "" {
gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL))
@ -56,25 +49,6 @@ func New(options *Options) (*Integration, error) {
return &Integration{client: git, userID: user.ID, options: options}, nil
}
func validateOptions(options *Options) error {
errs := []string{}
if options.Username == "" {
errs = append(errs, "Username")
}
if options.Token == "" {
errs = append(errs, "Token")
}
if options.ProjectName == "" {
errs = append(errs, "ProjectName")
}
if len(errs) > 0 {
return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
}
return nil
}
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)

40
v2/pkg/reporting/trackers/jira/jira.go
View File

@ -2,7 +2,6 @@ package jira
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"strings"
@ -28,15 +27,15 @@ type Options struct {
// UpdateExisting value (optional) if true, the existing opened issue is updated
UpdateExisting bool `yaml:"update-existing"`
// URL is the URL of the jira server
URL string `yaml:"url"`
URL string `yaml:"url" validate:"required"`
// AccountID is the accountID of the jira user.
AccountID string `yaml:"account-id"`
AccountID string `yaml:"account-id" validate:"required"`
// Email is the email of the user for jira instance
Email string `yaml:"email"`
Email string `yaml:"email" validate:"required,email"`
// Token is the token for jira instance.
Token string `yaml:"token"`
Token string `yaml:"token" validate:"required"`
// ProjectName is the name of the project.
ProjectName string `yaml:"project-name"`
ProjectName string `yaml:"project-name" validate:"required"`
// IssueType (optional) is the name of the created issue type
IssueType string `yaml:"issue-type"`
// SeverityAsLabel (optional) sends the severity as the label of the created
@ -46,10 +45,6 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
err := validateOptions(options)
if err != nil {
return nil, err
}
username := options.Email
if !options.Cloud {
username = options.AccountID
@ -65,31 +60,6 @@ func New(options *Options) (*Integration, error) {
return &Integration{jira: jiraClient, options: options}, nil
}
func validateOptions(options *Options) error {
errs := []string{}
if options.URL == "" {
errs = append(errs, "URL")
}
if options.AccountID == "" {
errs = append(errs, "AccountID")
}
if options.Email == "" {
errs = append(errs, "Email")
}
if options.Token == "" {
errs = append(errs, "Token")
}
if options.ProjectName == "" {
errs = append(errs, "ProjectName")
}
if len(errs) > 0 {
return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ","))
}
return nil
}
// CreateNewIssue creates a new issue in the tracker
func (i *Integration) CreateNewIssue(event *output.ResultEvent) error {
summary := format.Summary(event)

2
v2/pkg/types/types.go
View File

@ -49,7 +49,7 @@ type Options struct {
// ProjectPath allows nuclei to use a user defined project folder
ProjectPath string
// InteractshURL is the URL for the interactsh server.
InteractshURL string
InteractshURL string `validate:"omitempty,url"`
// Interactsh Authorization header value for self-hosted servers
InteractshToken string
// Target URLs/Domains to scan using a template

34
v2/pkg/utils/yaml/yaml_decode_wrapper.go Normal file
View File

@ -0,0 +1,34 @@
package yaml
import (
"io"
"strings"
"github.com/go-playground/validator/v10"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
var validate *validator.Validate
// DecodeAndValidate is a wrapper for yaml Decode adding struct validation
func DecodeAndValidate(r io.Reader, v interface{}) error {
if err := yaml.NewDecoder(r).Decode(v); err != nil {
return err
}
if validate == nil {
validate = validator.New()
}
if err := validate.Struct(v); err != nil {
if _, ok := err.(*validator.InvalidValidationError); ok {
return err
}
errs := []string{}
for _, err := range err.(validator.ValidationErrors) {
errs = append(errs, err.Namespace()+": "+err.Tag())
}
return errors.Wrap(errors.New(strings.Join(errs, ", ")), "validation failed for these fields")
}
return nil
}