Loader rewriter working poc

This commit is contained in:
Ice3man543 2021-07-01 14:36:40 +05:30
parent 7669e9781a
commit dff76e9cd2
9 changed files with 210 additions and 174 deletions

View File

@ -81,13 +81,6 @@ func validateOptions(options *types.Options) error {
return errors.New("both verbose and silent mode specified")
}
if !options.TemplateList {
// Check if a list of templates was provided and it exists
if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
}
// Validate proxy options if provided
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
if err != nil {

View File

@ -260,6 +260,17 @@ func (r *Runner) RunEnumeration() {
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
}
loaderConfig := &loader.Config{
Templates: r.options.Templates,
Workflows: r.options.Workflows,
@ -272,21 +283,60 @@ func (r *Runner) RunEnumeration() {
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Catalog: r.catalog,
ExecutorOptions: executerOpts,
}
store, err := loader.New(loaderConfig)
if err != nil {
gologger.Fatal().Msgf("Could not load templates from config: %s\n", err)
}
store.Load()
builder := &strings.Builder{}
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
builder.WriteString(" (")
if config.Version == r.templatesConfig.NucleiLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr := builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
builder.WriteString(" (")
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr = builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
if r.interactsh != nil {
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
}
if len(store.Templates()) > 0 {
gologger.Info().Msgf("Running Nuclei Templates (%d)", len(store.Templates()))
}
if len(store.Workflows()) > 0 {
gologger.Info().Msgf("Running Nuclei Workflows (%d)", len(store.Workflows()))
}
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows)
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
var unclusteredRequests int64 = 0
for _, template := range availableTemplates {
for _, template := range store.Templates() {
// workflows will dynamically adjust the totals while running, as
// it can't be know in advance which requests will be called
if len(template.Workflows) > 0 {
@ -295,9 +345,21 @@ func (r *Runner) RunEnumeration() {
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
}
originalTemplatesCount := len(availableTemplates)
if r.options.Verbose {
for _, template := range store.Templates() {
r.logAvailableTemplate(template.Path)
}
for _, template := range store.Workflows() {
r.logAvailableTemplate(template.Path)
}
}
templatesMap := make(map[string]*templates.Template)
for _, v := range store.Templates() {
templatesMap[v.ID] = v
}
originalTemplatesCount := len(store.Templates())
clusterCount := 0
clusters := clusterer.Cluster(availableTemplates)
clusters := clusterer.Cluster(templatesMap)
for _, cluster := range clusters {
if len(cluster) > 1 && !r.options.OfflineHTTP {
executerOpts := protocols.ExecuterOptions{
@ -324,7 +386,7 @@ func (r *Runner) RunEnumeration() {
finalTemplates = append(finalTemplates, cluster...)
}
}
for _, workflows := range availableWorkflows {
for _, workflows := range store.Workflows() {
finalTemplates = append(finalTemplates, workflows)
}
@ -336,20 +398,16 @@ func (r *Runner) RunEnumeration() {
totalRequests += int64(t.TotalRequests) * r.inputCount
}
if totalRequests < unclusteredRequests {
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
gologger.Info().Msgf("Reduced %d requests (%d templates clustered)", unclusteredRequests-totalRequests, clusterCount)
}
templateCount := originalTemplatesCount + len(availableWorkflows)
workflowCount := len(store.Workflows())
templateCount := originalTemplatesCount + workflowCount
// 0 matches means no templates were found in directory
if templateCount == 0 {
gologger.Fatal().Msgf("Error, no templates were found.\n")
}
gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
r.colorizer.Bold(templateCount).String(),
r.colorizer.Bold(templateCount-workflowCount).String(),
r.colorizer.Bold(workflowCount).String())
results := &atomic.Bool{}
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)

View File

@ -1,36 +1,36 @@
package runner
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/karrick/godirwalk"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"gopkg.in/yaml.v2"
)
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
}
template, err := templates.Parse(file, executerOpts)
f, err := os.Open(file)
if err != nil {
return nil, err
}
if template == nil {
return nil, nil
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return nil, err
}
return template, nil
}
@ -52,7 +52,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) {
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
}
}
@ -89,38 +89,12 @@ func (r *Runner) listAvailableTemplates() {
}
}
func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
for _, s := range allowedSeverities {
finalSeverities := []string{}
if strings.Contains(s, ",") {
finalSeverities = strings.Split(s, ",")
} else {
finalSeverities = append(finalSeverities, s)
}
for _, sev := range finalSeverities {
sev = strings.ToLower(sev)
if sev != "" && strings.HasPrefix(templateSeverity, sev) {
return true
}
}
}
return false
}
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
err := godirwalk.Walk(fsPath, &godirwalk.Options{
return godirwalk.Walk(fsPath, &godirwalk.Options{
Callback: callback,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Unsorted: true,
})
// directory couldn't be walked
if err != nil {
return err
}
return nil
}

View File

@ -7,6 +7,7 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@ -14,6 +15,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@ -23,6 +25,7 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
)
const (
@ -30,6 +33,13 @@ const (
repoName = "nuclei-templates"
)
const nucleiIgnoreFile = ".nuclei-ignore"
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// updateTemplates checks if the default list of nuclei-templates
// exist in the users home directory, if not the latest revision
// is downloaded from github.
@ -46,7 +56,7 @@ func (r *Runner) updateTemplates() error {
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
config, readErr := readConfiguration()
config, readErr := config.ReadConfiguration()
if err != nil {
return readErr
}
@ -55,12 +65,12 @@ func (r *Runner) updateTemplates() error {
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
if r.templatesConfig == nil {
currentConfig := &nucleiConfig{
currentConfig := &config.Config{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
IgnoreURL: ignoreURL,
NucleiVersion: Version,
NucleiVersion: config.Version,
}
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
return errors.Wrap(writeErr, "could not write template configuration")
}
r.templatesConfig = currentConfig
@ -68,12 +78,19 @@ func (r *Runner) updateTemplates() error {
// Check if last checked for nuclei-ignore is more than 1 hours.
// and if true, run the check.
//
// Also at the same time fetch latest version from github to do outdated nuclei
// and templates check.
checkedIgnore := false
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
r.fetchLatestVersionsFromGithub()
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
ignoreURL = r.templatesConfig.IgnoreURL
}
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
checkedIgnore = true
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
if reqErr == nil {
@ -106,7 +123,7 @@ func (r *Runner) updateTemplates() error {
}
// Use custom location if user has given a template directory
r.templatesConfig = &nucleiConfig{
r.templatesConfig = &config.Config{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
}
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
@ -120,13 +137,14 @@ func (r *Runner) updateTemplates() error {
}
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub() // also fetch latest versions
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
r.templatesConfig.CurrentVersion = version.String()
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
@ -162,13 +180,13 @@ func (r *Runner) updateTemplates() error {
if version.EQ(oldVersion) {
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
return r.writeConfiguration(r.templatesConfig)
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
}
if version.GT(oldVersion) {
if !r.options.UpdateTemplates {
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
return r.writeConfiguration(r.templatesConfig)
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
}
if r.options.TemplatesDirectory != "" {
@ -177,11 +195,12 @@ func (r *Runner) updateTemplates() error {
r.templatesConfig.CurrentVersion = version.String()
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub()
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
@ -461,3 +480,55 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
}
table.Render()
}
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
func (r *Runner) fetchLatestVersionsFromGithub() {
nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
}
templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
}
if r.templatesConfig != nil {
r.templatesConfig.NucleiLatestVersion = nucleiLatest
r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
}
}
type githubTagData struct {
Name string
}
// githubFetchLatestTagRepo fetches latest tag from github
func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
var tags []githubTagData
err = json.Unmarshal(body, &tags)
if err != nil {
return "", err
}
if len(tags) == 0 {
return "", fmt.Errorf("no tags found for %s", repo)
}
return tags[0].Name, nil
}

View File

@ -43,7 +43,6 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
defer os.RemoveAll(templatesDirectory)
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
require.Nil(t, err, "could not download release and unzip")
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")

View File

@ -3,7 +3,6 @@ package config
import (
"os"
"path"
"regexp"
"time"
jsoniter "github.com/json-iterator/go"
@ -19,13 +18,14 @@ type Config struct {
IgnoreURL string `json:"ignore-url,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
NucleiLatestVersion string `json:"nuclei-latest-version"`
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
}
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// Version is the current version of nuclei
const Version = `2.3.8`
@ -59,12 +59,16 @@ func ReadConfiguration() (*Config, error) {
}
// WriteConfiguration writes the updated nuclei configuration to disk
func WriteConfiguration(config *Config) error {
func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
if config.IgnoreURL == "" {
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
}
config.LastChecked = time.Now()
config.LastCheckedIgnore = time.Now()
if checked {
config.LastChecked = time.Now()
}
if checkedIgnore {
config.LastCheckedIgnore = time.Now()
}
config.NucleiVersion = Version
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
if err != nil {

View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"gopkg.in/yaml.v2"
@ -28,6 +29,7 @@ type Config struct {
IncludeTags []string
Catalog *catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
TemplatesDirectory string
}
@ -37,6 +39,9 @@ type Store struct {
config *Config
finalTemplates []string
templateMatched bool
templates []*templates.Template
workflows []*templates.Template
}
// New creates a new template store based on provided configuration
@ -47,7 +52,6 @@ func New(config *Config) (*Store, error) {
tagFilter: config.createTagFilter(),
}
// hasFindByMetadata := config.hasFindByMetadata()
// Handle a case with no templates or workflows, where we use base directory
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
config.Templates = append(config.Templates, config.TemplatesDirectory)
@ -55,13 +59,24 @@ func New(config *Config) (*Store, error) {
store.templateMatched = true
}
store.finalTemplates = append(store.finalTemplates, config.Templates...)
return store, nil
}
// Templates returns all the templates in the store
func (s *Store) Templates() []*templates.Template {
return s.templates
}
// Workflows returns all the workflows in the store
func (s *Store) Workflows() []*templates.Template {
return s.workflows
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (s *Store) Load() (templates []string, workflows []string) {
includedTemplates := s.config.Catalog.GetTemplatesPath(s.config.Templates)
func (s *Store) Load() {
includedTemplates := s.config.Catalog.GetTemplatesPath(s.finalTemplates)
includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates)
alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates)
@ -89,7 +104,12 @@ func (s *Store) Load() (templates []string, workflows []string) {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
}
if loaded {
templates = append(templates, k)
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
} else if parsed != nil {
s.templates = append(s.templates, parsed)
}
}
}
@ -110,10 +130,14 @@ func (s *Store) Load() (templates []string, workflows []string) {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
}
if loaded {
workflows = append(workflows, k)
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
} else if parsed != nil {
s.workflows = append(s.workflows, parsed)
}
}
}
return templates, workflows
}
// loadTemplateParseMetadata loads a template by parsing metadata and running

View File

@ -189,49 +189,3 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
}
return nil
}
// matchTemplateWithTags matches if the template matches a tag
func matchTemplateWithTags(tags, severity string, tagsInput []string) error {
actualTags := strings.Split(tags, ",")
if severity != "" {
actualTags = append(actualTags, severity) // also add severity to tag
}
matched := false
mainLoop:
for _, t := range tagsInput {
commaTags := strings.Split(t, ",")
for _, tag := range commaTags {
tag = strings.TrimSpace(tag)
key, value := getKeyValue(tag)
for _, templTag := range actualTags {
templTag = strings.TrimSpace(templTag)
tKey, tValue := getKeyValue(templTag)
if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
matched = true
break mainLoop
}
}
}
}
if !matched {
return errors.New("could not match template tags with input")
}
return nil
}
// getKeyValue returns key value pair for a data string
func getKeyValue(data string) (key, value string) {
if strings.Contains(data, ":") {
parts := strings.SplitN(data, ":", 2)
if len(parts) == 2 {
key, value = parts[0], parts[1]
}
}
if value == "" {
value = data
}
return key, value
}

View File

@ -1,41 +0,0 @@
package templates
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMatchTemplateWithTags(t *testing.T) {
err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"})
require.Nil(t, err, "could not get php tag from input slice")
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"})
require.Nil(t, err, "could not get php tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"})
require.NotNil(t, err, "could get key value tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"})
require.NotNil(t, err, "could get key value tag from input key value")
t.Run("space", func(t *testing.T) {
err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"})
require.Nil(t, err, "could get key value tag from input key value with space")
})
t.Run("comma-tags", func(t *testing.T) {
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"})
require.Nil(t, err, "could get key value tag from input key value with comma")
})
t.Run("severity", func(t *testing.T) {
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"})
require.Nil(t, err, "could get key value tag for severity")
})
t.Run("blank-tags", func(t *testing.T) {
err = matchTemplateWithTags("", "low", []string{"jira"})
require.NotNil(t, err, "could get value tag for blank severity")
})
}