Misc

v2/pkg/output/output.go

@@ -125,6 +125,9 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 	if err != nil {
 		return errors.Wrap(err, "could not format output")
 	}
+	if len(data) == 0 {
+		return nil
+	}
 	_, _ = os.Stdout.Write(data)
 	_, _ = os.Stdout.Write([]byte("\n"))
 	if w.outputFile != nil {
@@ -134,7 +137,6 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 		if writeErr := w.outputFile.Write(data); writeErr != nil {
 			return errors.Wrap(err, "could not write to output")
 		}
-		_ = w.outputFile.Write([]byte("\n"))
 	}
 	return nil
 }

v2/pkg/protocols/file/file.go

@@ -18,8 +18,6 @@ type Request struct {
 	NoRecursive bool `yaml:"no-recursive"`
 	// Extensions is the list of extensions to perform matching on.
 	Extensions []string `yaml:"extensions"`
-	// ExtensionAllowlist is the list of file extensions to enforce allowing.
-	ExtensionAllowlist []string `yaml:"allowlist"`
 	// ExtensionDenylist is the list of file extensions to deny during matching.
 	ExtensionDenylist []string `yaml:"denylist"`
 
@@ -73,9 +71,6 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
 	for _, extension := range r.ExtensionDenylist {
 		r.extensionDenylist[extension] = struct{}{}
 	}
-	for _, extension := range r.ExtensionAllowlist {
-		delete(r.extensionDenylist, extension)
-	}
 	return nil
 }
 

v2/pkg/protocols/file/file_test.go

@@ -13,12 +13,11 @@ func TestFileCompile(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/file/find.go

@@ -106,11 +106,11 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
 // validatePath validates a file path for blacklist and whitelist options
 func (r *Request) validatePath(item string) bool {
 	extension := path.Ext(item)
-	if len(r.extensions) > 0 && !r.allExtensions {
+
+	if len(r.extensions) > 0 {
 		if _, ok := r.extensions[extension]; ok {
 			return true
 		}
-		return false
 	}
 	if _, ok := r.extensionDenylist[extension]; ok {
 		gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)

v2/pkg/protocols/file/find_test.go

@@ -16,12 +16,11 @@ func TestFindInputPaths(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,

v2/pkg/protocols/file/operators_test.go

@@ -17,12 +17,11 @@ func TestResponseToDSLMap(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -43,12 +42,11 @@ func TestFileOperatorMatch(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -109,12 +107,11 @@ func TestFileOperatorExtract(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 	}
 	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
 		ID:   templateID,
@@ -162,12 +159,11 @@ func TestFileMakeResult(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*", ".lock"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 		Operators: operators.Operators{
 			Matchers: []*matchers.Matcher{{
 				Name:  "test",

v2/pkg/protocols/file/request_test.go

@@ -20,12 +20,11 @@ func TestFileExecuteWithResults(t *testing.T) {
 	testutils.Init(options)
 	templateID := "testing-file"
 	request := &Request{
-		ID:                 templateID,
+		ID:                templateID,
-		MaxSize:            1024,
+		MaxSize:           1024,
-		NoRecursive:        false,
+		NoRecursive:       false,
-		Extensions:         []string{"*"},
+		Extensions:        []string{"*"},
-		ExtensionAllowlist: []string{".lock"},
+		ExtensionDenylist: []string{".go"},
-		ExtensionDenylist:  []string{".go"},
 		Operators: operators.Operators{
 			Matchers: []*matchers.Matcher{{
 				Name:  "test",

v2/pkg/protocols/http/build_request.go

@@ -49,21 +49,23 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa
 	if err != nil {
 		return nil, err
 	}
+
+	data, parsed = baseURLWithTemplatePrefs(data, parsed)
+	values := generators.MergeMaps(dynamicValues, map[string]interface{}{
+		"Hostname": parsed.Host,
+	})
+
 	isRawRequest := strings.Contains(data, "\n")
 	if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") {
 		parsed.Path = strings.TrimSuffix(parsed.Path, "/")
 	}
-
+	parsedString := parsed.String()
-	hostname := parsed.Host
+	values["BaseURL"] = parsedString
-	values := generators.MergeMaps(dynamicValues, map[string]interface{}{
-		"BaseURL":  baseURLWithTemplatePrefs(data, parsed),
-		"Hostname": hostname,
-	})
 
 	// If data contains \n it's a raw request, process it like raw. Else
 	// continue with the template based request flow.
 	if isRawRequest {
-		return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values, payloads)
+		return r.makeHTTPRequestFromRaw(ctx, parsedString, data, values, payloads)
 	}
 	return r.makeHTTPRequestFromModel(ctx, data, values)
 }
@@ -78,15 +80,19 @@ func (r *requestGenerator) Total() int {
 
 // baseURLWithTemplatePrefs returns the url for BaseURL keeping
 // the template port and path preference over the user provided one.
-func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string {
+func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) {
-	// template port preference over input URL port
+	// template port preference over input URL port if template has a port
-	// template has port
+	matches := urlWithPortRegex.FindAllStringSubmatch(data, -1)
-	if urlWithPortRegex.MatchString(data) {
+	if len(matches) == 0 {
-		if _, port, err := net.SplitHostPort(data); err == nil {
+		return data, parsed
-			parsedURL.Host = net.JoinHostPort(parsedURL.Hostname(), port)
-		}
 	}
-	return parsedURL.String()
+	port := matches[0][1]
+	parsed.Host = net.JoinHostPort(parsed.Hostname(), port)
+	data = strings.ReplaceAll(data, ":"+port, "")
+	if parsed.Path == "" {
+		parsed.Path = "/"
+	}
+	return data, parsed
 }
 
 // MakeHTTPRequestFromModel creates a *http.Request from a request template

v2/pkg/protocols/http/build_request_test.go

@@ -9,10 +9,13 @@ import (
 )
 
 func TestBaseURLWithTemplatePrefs(t *testing.T) {
-	parsed, _ := url.Parse("http://localhost:53")
+	baseURL := "http://localhost:53/test"
+	parsed, _ := url.Parse(baseURL)
 
-	new := baseURLWithTemplatePrefs("{{BaseURL}}:8000", parsed)
+	data := "{{BaseURL}}:8000/newpath"
-	require.Equal(t, "http://localhost:8000", new, "could not get correct value")
+	data, new := baseURLWithTemplatePrefs(data, parsed)
+	require.Equal(t, "http://localhost:8000/test", new, "could not get correct value")
+	require.Equal(t, "{{BaseURL}}/newpath", data, "could not get correct data")
 }
 
 func TestMakeRequestFromModal(t *testing.T) {

v2/pkg/templates/compile.go

@@ -34,9 +34,6 @@ func Parse(filePath string, options *protocols.ExecuterOptions) (*Template, erro
 	if _, ok := template.Info["author"]; !ok {
 		return nil, errors.New("no template author field provided")
 	}
-	if _, ok := template.Info["severity"]; !ok {
-		return nil, errors.New("no template severity field provided")
-	}
 	if len(options.Options.Tags) > 0 {
 		templateTags, ok := template.Info["tags"]
 		if !ok {

v2/pkg/workflows/execute.go

@@ -1,6 +1,8 @@
 package workflows
 
 import (
+	"fmt"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"go.uber.org/atomic"
@@ -81,6 +83,8 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 				continue
 			}
 			if executionErr != nil {
+
+				fmt.Printf("%+v\n", executionErr)
 				if len(template.Executers) == 1 {
 					mainErr = executionErr
 				} else {